From 8721e528add28aefae2f75615ded74d29ac361f5 Mon Sep 17 00:00:00 2001 From: zml1206 Date: Tue, 18 Jun 2024 13:05:57 +0800 Subject: [PATCH] update --- .../tpch-approved-plan/v1-ras/spark32/10.txt | 356 ++++----- .../tpch-approved-plan/v1-ras/spark32/11.txt | 282 +++---- .../tpch-approved-plan/v1-ras/spark32/12.txt | 180 ++--- .../tpch-approved-plan/v1-ras/spark32/13.txt | 194 +++-- .../tpch-approved-plan/v1-ras/spark32/14.txt | 132 ++- .../tpch-approved-plan/v1-ras/spark32/15.txt | 168 ++-- .../tpch-approved-plan/v1-ras/spark32/16.txt | 236 +++--- .../tpch-approved-plan/v1-ras/spark32/17.txt | 225 +++--- .../tpch-approved-plan/v1-ras/spark32/18.txt | 407 +++++----- .../tpch-approved-plan/v1-ras/spark32/19.txt | 128 ++- .../tpch-approved-plan/v1-ras/spark32/20.txt | 534 ++++++------- .../tpch-approved-plan/v1-ras/spark32/21.txt | 499 ++++++------ .../tpch-approved-plan/v1-ras/spark32/22.txt | 168 ++-- .../tpch-approved-plan/v1-ras/spark32/3.txt | 238 +++--- .../tpch-approved-plan/v1-ras/spark32/4.txt | 184 ++--- .../tpch-approved-plan/v1-ras/spark32/5.txt | 560 ++++++------- .../tpch-approved-plan/v1-ras/spark32/7.txt | 538 ++++++------- .../tpch-approved-plan/v1-ras/spark32/8.txt | 752 ++++++++---------- .../tpch-approved-plan/v1-ras/spark32/9.txt | 556 ++++++------- .../tpch-approved-plan/v1-ras/spark33/10.txt | 356 ++++----- .../tpch-approved-plan/v1-ras/spark33/11.txt | 502 ++++++------ .../tpch-approved-plan/v1-ras/spark33/12.txt | 180 ++--- .../tpch-approved-plan/v1-ras/spark33/13.txt | 194 +++-- .../tpch-approved-plan/v1-ras/spark33/14.txt | 132 ++- .../tpch-approved-plan/v1-ras/spark33/15.txt | 256 +++--- .../tpch-approved-plan/v1-ras/spark33/16.txt | 236 +++--- .../tpch-approved-plan/v1-ras/spark33/17.txt | 225 +++--- .../tpch-approved-plan/v1-ras/spark33/18.txt | 407 +++++----- .../tpch-approved-plan/v1-ras/spark33/19.txt | 128 ++- .../tpch-approved-plan/v1-ras/spark33/20.txt | 526 ++++++------ .../tpch-approved-plan/v1-ras/spark33/21.txt | 495 ++++++------ .../tpch-approved-plan/v1-ras/spark33/22.txt | 238 +++--- .../tpch-approved-plan/v1-ras/spark33/3.txt | 238 +++--- .../tpch-approved-plan/v1-ras/spark33/4.txt | 184 ++--- .../tpch-approved-plan/v1-ras/spark33/5.txt | 560 ++++++------- .../tpch-approved-plan/v1-ras/spark33/7.txt | 538 ++++++------- .../tpch-approved-plan/v1-ras/spark33/8.txt | 752 ++++++++---------- .../tpch-approved-plan/v1-ras/spark33/9.txt | 556 ++++++------- .../tpch-approved-plan/v1-ras/spark34/10.txt | 356 ++++----- .../tpch-approved-plan/v1-ras/spark34/11.txt | 502 ++++++------ .../tpch-approved-plan/v1-ras/spark34/12.txt | 180 ++--- .../tpch-approved-plan/v1-ras/spark34/13.txt | 194 +++-- .../tpch-approved-plan/v1-ras/spark34/14.txt | 132 ++- .../tpch-approved-plan/v1-ras/spark34/15.txt | 256 +++--- .../tpch-approved-plan/v1-ras/spark34/16.txt | 236 +++--- .../tpch-approved-plan/v1-ras/spark34/17.txt | 225 +++--- .../tpch-approved-plan/v1-ras/spark34/18.txt | 407 +++++----- .../tpch-approved-plan/v1-ras/spark34/19.txt | 128 ++- .../tpch-approved-plan/v1-ras/spark34/20.txt | 526 ++++++------ .../tpch-approved-plan/v1-ras/spark34/21.txt | 495 ++++++------ .../tpch-approved-plan/v1-ras/spark34/22.txt | 238 +++--- .../tpch-approved-plan/v1-ras/spark34/3.txt | 238 +++--- .../tpch-approved-plan/v1-ras/spark34/4.txt | 184 ++--- .../tpch-approved-plan/v1-ras/spark34/5.txt | 560 ++++++------- .../tpch-approved-plan/v1-ras/spark34/7.txt | 538 ++++++------- .../tpch-approved-plan/v1-ras/spark34/8.txt | 752 ++++++++---------- .../tpch-approved-plan/v1-ras/spark34/9.txt | 556 ++++++------- .../sql/execution/adaptive/GlutenCost.scala | 43 + .../adaptive/GlutenCostEvaluator.scala | 15 +- .../adaptive/GlutenCostEvaluator.scala | 17 +- .../adaptive/GlutenCostEvaluator.scala | 17 +- .../adaptive/GlutenCostEvaluator.scala | 17 +- 62 files changed, 9126 insertions(+), 10726 deletions(-) create mode 100644 shims/common/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCost.scala diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt index 151861eb5766..25741f30232b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt @@ -1,90 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- TakeOrderedAndProjectExecTransformer (65) - +- ^ ProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59) - +- ColumnarExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ FlushableHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - :- ^ SortExecTransformer (43) - : +- ^ InputIteratorTransformer (42) - : +- ShuffleQueryStage (40) - : +- ColumnarExchange (39) - : +- ^ ProjectExecTransformer (37) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : :- ^ SortExecTransformer (26) - : : +- ^ InputIteratorTransformer (25) - : : +- ShuffleQueryStage (23) - : : +- ColumnarExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : : :- ^ SortExecTransformer (9) - : : : +- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ SortExecTransformer (18) - : : +- ^ InputIteratorTransformer (17) - : : +- ShuffleQueryStage (15) - : : +- ColumnarExchange (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ NoopFilter (11) - : : +- ^ Scan parquet (10) - : +- ^ SortExecTransformer (35) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ NoopFilter (28) - : +- ^ Scan parquet (27) - +- ^ SortExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ NoopFilter (45) - +- ^ Scan parquet (44) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + :- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : :- ^ InputIteratorTransformer (23) + : : +- ShuffleQueryStage (21) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ShuffleQueryStage (29) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ShuffleQueryStage (44) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- SortMergeJoin Inner (93) - :- Sort (88) - : +- Exchange (87) - : +- Project (86) - : +- SortMergeJoin Inner (85) - : :- Sort (79) - : : +- Exchange (78) - : : +- Project (77) - : : +- SortMergeJoin Inner (76) - : : :- Sort (70) - : : : +- Exchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- Sort (75) - : : +- Exchange (74) - : : +- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Sort (84) - : +- Exchange (83) - : +- Project (82) - : +- Filter (81) - : +- Scan parquet (80) - +- Sort (92) - +- Exchange (91) - +- Filter (90) - +- Scan parquet (89) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- SortMergeJoin Inner (87) + :- Sort (82) + : +- Exchange (81) + : +- Project (80) + : +- SortMergeJoin Inner (79) + : :- Sort (73) + : : +- Exchange (72) + : : +- Project (71) + : : +- SortMergeJoin Inner (70) + : : :- Sort (64) + : : : +- Exchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- Sort (69) + : : +- Exchange (68) + : : +- Project (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Sort (78) + : +- Exchange (77) + : +- Project (76) + : +- Filter (75) + : +- Scan parquet (74) + +- Sort (86) + +- Exchange (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -120,392 +114,368 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct (8) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) SortExecTransformer -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(18) SortExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(24) InputAdapter -Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(27) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(35) SortExecTransformer -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(41) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(42) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(43) SortExecTransformer -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 - -(44) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(50) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(51) InputIteratorTransformer +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(52) SortExecTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(55) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) ProjectExecTransformer +(50) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(57) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(58) ColumnarExchange +(52) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(53) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(60) InputAdapter +(54) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(61) InputIteratorTransformer +(55) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(62) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(63) ProjectExecTransformer +(57) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(64) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(65) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(66) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) Scan parquet +(61) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(62) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) Exchange +(63) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) Sort +(64) Sort Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(71) Scan parquet +(65) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(72) Filter +(66) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(73) Project +(67) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(74) Exchange +(68) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) Sort +(69) Sort Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(76) SortMergeJoin +(70) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(77) Project +(71) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(78) Exchange +(72) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Sort +(73) Sort Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(80) Scan parquet +(74) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(81) Filter +(75) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(82) Project +(76) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(83) Exchange +(77) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Sort +(78) Sort Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(85) SortMergeJoin +(79) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(86) Project +(80) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(87) Exchange +(81) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Sort +(82) Sort Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(89) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(90) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(91) Exchange +(85) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(86) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(93) SortMergeJoin +(87) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(94) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(95) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(96) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(98) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(99) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt index 76a2004365ff..ce24f02151f9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt @@ -1,73 +1,69 @@ == Physical Plan == -AdaptiveSparkPlan (80) +AdaptiveSparkPlan (76) +- == Final Plan == - VeloxColumnarToRowExec (54) - +- ^ SortExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ FilterExecTransformer (46) - +- ^ RegularHashAggregateExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ShuffleQueryStage (42) - +- ColumnarExchange (41) - +- ^ ProjectExecTransformer (39) - +- ^ FlushableHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - :- ^ SortExecTransformer (26) - : +- ^ InputIteratorTransformer (25) - : +- ShuffleQueryStage (23) - : +- ColumnarExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ NoopFilter (11) - : +- ^ Scan parquet (10) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32) - +- ColumnarExchange (31) - +- ^ ProjectExecTransformer (29) - +- ^ NoopFilter (28) - +- ^ Scan parquet (27) + VeloxColumnarToRowExec (50) + +- ^ SortExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ShuffleQueryStage (45) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ShuffleQueryStage (38) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + :- ^ InputIteratorTransformer (23) + : +- ShuffleQueryStage (21) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - Sort (79) - +- Exchange (78) - +- Filter (77) - +- HashAggregate (76) - +- Exchange (75) - +- HashAggregate (74) - +- Project (73) - +- SortMergeJoin Inner (72) - :- Sort (66) - : +- Exchange (65) - : +- Project (64) - : +- SortMergeJoin Inner (63) - : :- Sort (58) - : : +- Exchange (57) - : : +- Filter (56) - : : +- Scan parquet (55) - : +- Sort (62) - : +- Exchange (61) - : +- Filter (60) - : +- Scan parquet (59) - +- Sort (71) - +- Exchange (70) - +- Project (69) - +- Filter (68) - +- Scan parquet (67) + Sort (75) + +- Exchange (74) + +- Filter (73) + +- HashAggregate (72) + +- Exchange (71) + +- HashAggregate (70) + +- Project (69) + +- SortMergeJoin Inner (68) + :- Sort (62) + : +- Exchange (61) + : +- Project (60) + : +- SortMergeJoin Inner (59) + : :- Sort (54) + : : +- Exchange (53) + : : +- Filter (52) + : : +- Scan parquet (51) + : +- Sort (58) + : +- Exchange (57) + : +- Filter (56) + : +- Scan parquet (55) + +- Sort (67) + +- Exchange (66) + +- Project (65) + +- Filter (64) + +- Scan parquet (63) (1) Scan parquet @@ -103,310 +99,294 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] (8) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) SortExecTransformer -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(18) SortExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(26) SortExecTransformer -Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(27) Scan parquet +(24) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(33) InputAdapter -Input [1]: [n_nationkey#X] - -(34) InputIteratorTransformer +(30) InputAdapter Input [1]: [n_nationkey#X] -(35) SortExecTransformer +(31) InputIteratorTransformer Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(38) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(39) ProjectExecTransformer +(35) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(41) ColumnarExchange +(37) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(38) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(43) InputAdapter +(39) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(44) InputIteratorTransformer +(40) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(45) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(46) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(47) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(48) ColumnarExchange +(44) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(50) InputAdapter +(46) InputAdapter Input [2]: [ps_partkey#X, value#X] -(51) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(52) SortExecTransformer +(48) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(53) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(54) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(55) Scan parquet +(51) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(56) Filter +(52) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(57) Exchange +(53) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) Sort +(54) Sort Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 -(59) Scan parquet +(55) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(56) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) Exchange +(57) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) Sort +(58) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(63) SortMergeJoin +(59) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(64) Project +(60) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(65) Exchange +(61) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Sort +(62) Sort Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(67) Scan parquet +(63) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(68) Filter +(64) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(69) Project +(65) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(70) Exchange +(66) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(67) Sort Input [1]: [n_nationkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(72) SortMergeJoin +(68) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(73) Project +(69) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(74) HashAggregate +(70) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(75) Exchange +(71) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(77) Filter +(73) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(78) Exchange +(74) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Sort +(75) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(80) AdaptiveSparkPlan +(76) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt index 43bc59ec21d7..85b5fee0725f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt @@ -1,50 +1,48 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (51) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- SortMergeJoin Inner (46) - :- Sort (40) - : +- Exchange (39) - : +- Filter (38) - : +- Scan parquet (37) - +- Sort (45) - +- Exchange (44) - +- Project (43) - +- Filter (42) - +- Scan parquet (41) + Sort (50) + +- Exchange (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- Project (45) + +- SortMergeJoin Inner (44) + :- Sort (38) + : +- Exchange (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Sort (43) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -80,198 +78,190 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] (8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) SortExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(18) SortExecTransformer -Input [2]: [l_orderkey#X, l_shipmode#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(21) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(24) ColumnarExchange +(22) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(26) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(27) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(28) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(29) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(30) ColumnarExchange +(28) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(32) InputAdapter +(30) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(33) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(34) SortExecTransformer +(32) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(36) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(37) Scan parquet +(35) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(38) Filter +(36) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(39) Exchange +(37) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(38) Sort Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(41) Scan parquet +(39) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(42) Filter +(40) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(43) Project +(41) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(44) Exchange +(42) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(43) Sort Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(46) SortMergeJoin +(44) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(47) Project +(45) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(48) HashAggregate +(46) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(49) Exchange +(47) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(48) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(51) Exchange +(49) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(50) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(51) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt index 3e5d236a05b5..8d2b2aceb2e8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt @@ -1,53 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) - :- ^ SortExecTransformer (8) - : +- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) + :- ^ InputIteratorTransformer (7) + : +- ShuffleQueryStage (5) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- Project (48) - +- SortMergeJoin LeftOuter (47) - :- Sort (41) - : +- Exchange (40) - : +- Scan parquet (39) - +- Sort (46) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftOuter (45) + :- Sort (39) + : +- Exchange (38) + : +- Scan parquet (37) + +- Sort (44) + +- Exchange (43) + +- Project (42) + +- Filter (41) + +- Scan parquet (40) (1) Scan parquet @@ -78,225 +76,217 @@ Input [1]: [c_custkey#X] (7) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) SortExecTransformer -Input [1]: [c_custkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(9) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(10) NoopFilter +(9) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) ProjectExecTransformer +(10) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(12) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(12) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(13) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(14) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(15) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) SortExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(18) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(19) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(20) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(21) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(23) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(24) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(25) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(26) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(28) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(29) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(30) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(31) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(32) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(34) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(35) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(36) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(38) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(39) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(40) Exchange +(38) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(39) Sort Input [1]: [c_custkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(42) Scan parquet +(40) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(43) Filter +(41) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(44) Project +(42) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(45) Exchange +(43) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(44) Sort Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(47) SortMergeJoin +(45) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(49) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(50) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(51) HashAggregate +(49) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(52) Exchange +(50) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(51) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(54) Exchange +(52) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(53) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(56) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt index cda1f5cdac58..6682914572d7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt @@ -1,40 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- ^ ProjectExecTransformer (23) - +- ^ RegularHashAggregateExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (38) - +- HashAggregate (37) - +- Project (36) - +- SortMergeJoin Inner (35) - :- Sort (30) - : +- Exchange (29) - : +- Project (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Sort (34) - +- Exchange (33) - +- Filter (32) - +- Scan parquet (31) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- SortMergeJoin Inner (33) + :- Sort (28) + : +- Exchange (27) + : +- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- Sort (32) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -70,146 +68,138 @@ Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] (8) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) SortExecTransformer -Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X, p_type#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(18) SortExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(21) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(22) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(23) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(24) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(25) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(26) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(27) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(28) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(29) Exchange +(27) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) Sort +(28) Sort Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(31) Scan parquet +(29) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(32) Filter +(30) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(33) Exchange +(31) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Sort +(32) Sort Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(35) SortMergeJoin +(33) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(36) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(37) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(38) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(39) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt index 203df8978486..8e375f6dff8e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt @@ -1,48 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (47) +- == Final Plan == - VeloxColumnarToRowExec (32) - +- ^ SortExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (22) - +- ^ FilterExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- Project (46) - +- SortMergeJoin Inner (45) - :- Sort (36) - : +- Exchange (35) - : +- Filter (34) - : +- Scan parquet (33) - +- Sort (44) - +- Filter (43) - +- HashAggregate (42) - +- Exchange (41) - +- HashAggregate (40) - +- Project (39) - +- Filter (38) - +- Scan parquet (37) + Sort (46) + +- Exchange (45) + +- Project (44) + +- SortMergeJoin Inner (43) + :- Sort (34) + : +- Exchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Sort (42) + +- Filter (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- Filter (36) + +- Scan parquet (35) (1) Scan parquet @@ -78,184 +76,176 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] (8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) SortExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(13) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) ProjectExecTransformer +(13) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(15) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(16) ColumnarExchange +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(18) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(21) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(22) SortExecTransformer -Input [2]: [supplier_no#X, total_revenue#X] -Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 - -(23) ShuffledHashJoinExecTransformer +(21) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(24) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(25) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(26) ColumnarExchange +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(28) InputAdapter +(26) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(29) InputIteratorTransformer +(27) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) SortExecTransformer +(28) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(31) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(32) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(33) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(34) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(35) Exchange +(33) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) Sort +(34) Sort Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(37) Scan parquet +(35) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(38) Filter +(36) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(39) Project +(37) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(40) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(41) Exchange +(39) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) HashAggregate +(40) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(43) Filter +(41) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(44) Sort +(42) Sort Input [2]: [supplier_no#X, total_revenue#X] Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 -(45) SortMergeJoin +(43) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(46) Project +(44) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(47) Exchange +(45) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(46) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(47) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt index 8a19de99eb06..5164125ce6e1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt @@ -1,63 +1,61 @@ == Physical Plan == -AdaptiveSparkPlan (68) +AdaptiveSparkPlan (66) +- == Final Plan == - VeloxColumnarToRowExec (44) - +- ^ SortExecTransformer (42) - +- ^ InputIteratorTransformer (41) - +- ShuffleQueryStage (39) - +- ColumnarExchange (38) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ ProjectExecTransformer (30) - +- ^ FlushableHashAggregateExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (42) + +- ^ SortExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (67) - +- Exchange (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- HashAggregate (62) - +- Exchange (61) - +- HashAggregate (60) - +- Project (59) - +- SortMergeJoin Inner (58) - :- Sort (53) - : +- Exchange (52) - : +- BroadcastHashJoin LeftAnti BuildRight (51) - : :- Filter (46) - : : +- Scan parquet (45) - : +- BroadcastExchange (50) - : +- Project (49) - : +- Filter (48) - : +- Scan parquet (47) - +- Sort (57) - +- Exchange (56) - +- Filter (55) - +- Scan parquet (54) + Sort (65) + +- Exchange (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Exchange (59) + +- HashAggregate (58) + +- Project (57) + +- SortMergeJoin Inner (56) + :- Sort (51) + : +- Exchange (50) + : +- BroadcastHashJoin LeftAnti BuildRight (49) + : :- Filter (44) + : : +- Scan parquet (43) + : +- BroadcastExchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Sort (55) + +- Exchange (54) + +- Filter (53) + +- Scan parquet (52) (1) Scan parquet @@ -93,272 +91,264 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] (8) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) SortExecTransformer -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(18) SortExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(21) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(24) ColumnarExchange +(22) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(23) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(26) InputAdapter +(24) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) InputIteratorTransformer +(25) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(28) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(29) FlushableHashAggregateExecTransformer +(27) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(30) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(32) ColumnarExchange +(30) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(34) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(35) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(36) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(37) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(38) ColumnarExchange +(36) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(37) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(40) InputAdapter +(38) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(41) InputIteratorTransformer +(39) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) SortExecTransformer +(40) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(43) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(44) VeloxColumnarToRowExec +(42) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(45) Scan parquet +(43) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(46) Filter +(44) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(47) Scan parquet +(45) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(48) Filter +(46) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(49) Project +(47) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(50) BroadcastExchange +(48) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(51) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(52) Exchange +(50) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) Sort +(51) Sort Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 -(54) Scan parquet +(52) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(55) Filter +(53) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(56) Exchange +(54) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(55) Sort Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(58) SortMergeJoin +(56) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(59) Project +(57) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(60) HashAggregate +(58) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(61) Exchange +(59) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(60) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(63) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(64) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(66) Exchange +(64) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(67) Sort +(65) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(68) AdaptiveSparkPlan +(66) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt index 4df858442c52..ad05a035acd3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt @@ -1,62 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ ProjectExecTransformer (38) - +- ^ RegularHashAggregateExecTransformer (37) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ ProjectExecTransformer (35) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) - :- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ NoopFilter (11) - : +- ^ Scan parquet (10) - +- ^ SortExecTransformer (33) - +- ^ FilterExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ NoopFilter (22) - +- ^ Scan parquet (21) + VeloxColumnarToRowExec (37) + +- ^ ProjectExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ RegularHashAggregateExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) + :- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ FilterExecTransformer (30) + +- ^ ProjectExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ NoopFilter (20) + +- ^ Scan parquet (19) +- == Initial Plan == - HashAggregate (62) - +- HashAggregate (61) - +- Project (60) - +- SortMergeJoin Inner (59) - :- Project (51) - : +- SortMergeJoin Inner (50) - : :- Sort (44) - : : +- Exchange (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Sort (49) - : +- Exchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Sort (58) - +- Filter (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Filter (53) - +- Scan parquet (52) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- SortMergeJoin Inner (56) + :- Project (48) + : +- SortMergeJoin Inner (47) + : :- Sort (41) + : : +- Exchange (40) + : : +- Filter (39) + : : +- Scan parquet (38) + : +- Sort (46) + : +- Exchange (45) + : +- Project (44) + : +- Filter (43) + : +- Scan parquet (42) + +- Sort (55) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -92,260 +89,248 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] (8) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) SortExecTransformer -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_container#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [1]: [p_partkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [p_partkey#X] -(18) SortExecTransformer -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(21) Scan parquet +(19) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(22) NoopFilter +(20) NoopFilter Input [2]: [l_partkey#X, l_quantity#X] Arguments: [l_partkey#X, l_quantity#X] -(23) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(24) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(25) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(26) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(28) InputAdapter +(26) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(29) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(30) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(31) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(32) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(33) SortExecTransformer -Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(34) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(35) ProjectExecTransformer +(32) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(36) RegularHashAggregateExecTransformer +(33) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(37) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(38) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(39) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(40) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(41) Scan parquet +(38) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(42) Filter +(39) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(43) Exchange +(40) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(41) Sort Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(45) Scan parquet +(42) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(46) Filter +(43) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(47) Project +(44) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(48) Exchange +(45) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(46) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(50) SortMergeJoin +(47) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(51) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(52) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(53) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(54) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(55) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(57) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(58) Sort +(55) Sort Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(59) SortMergeJoin +(56) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(60) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(61) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(62) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(63) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt index 2091add64d48..edbdea4771af 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt @@ -1,103 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (111) +AdaptiveSparkPlan (104) +- == Final Plan == - VeloxColumnarToRowExec (71) - +- TakeOrderedAndProjectExecTransformer (70) - +- ^ RegularHashAggregateExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (67) - +- ^ ProjectExecTransformer (66) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (65) - :- ^ SortExecTransformer (46) - : +- ^ InputIteratorTransformer (45) - : +- ShuffleQueryStage (43) - : +- ColumnarExchange (42) - : +- ^ ProjectExecTransformer (40) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (38) - : +- ^ InputIteratorTransformer (37) - : +- ShuffleQueryStage (35) - : +- ColumnarExchange (34) - : +- ^ ProjectExecTransformer (32) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) - : :- ^ SortExecTransformer (18) - : : +- ^ InputIteratorTransformer (17) - : : +- ShuffleQueryStage (15) - : : +- ColumnarExchange (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ NoopFilter (11) - : : +- ^ Scan parquet (10) - : +- ^ SortExecTransformer (30) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ RegularHashAggregateExecTransformer (27) - : +- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24) - : +- ColumnarExchange (23) - : +- ^ ProjectExecTransformer (21) - : +- ^ FlushableHashAggregateExecTransformer (20) - : +- ^ Scan parquet (19) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (64) - :- ^ SortExecTransformer (55) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ SortExecTransformer (63) - +- ^ ProjectExecTransformer (62) - +- ^ FilterExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ InputIteratorTransformer (59) - +- ShuffleQueryStage (57) - +- ReusedExchange (56) + VeloxColumnarToRowExec (64) + +- TakeOrderedAndProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) + :- ^ InputIteratorTransformer (41) + : +- ShuffleQueryStage (39) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : :- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ShuffleQueryStage (22) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) + :- ^ InputIteratorTransformer (49) + : +- ShuffleQueryStage (47) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (110) - +- HashAggregate (109) - +- HashAggregate (108) - +- Project (107) - +- SortMergeJoin Inner (106) - :- Sort (93) - : +- Exchange (92) - : +- Project (91) - : +- SortMergeJoin Inner (90) - : :- Sort (75) - : : +- Exchange (74) - : : +- Filter (73) - : : +- Scan parquet (72) - : +- Sort (89) - : +- Exchange (88) - : +- SortMergeJoin LeftSemi (87) - : :- Sort (79) - : : +- Exchange (78) - : : +- Filter (77) - : : +- Scan parquet (76) - : +- Sort (86) - : +- Project (85) - : +- Filter (84) - : +- HashAggregate (83) - : +- Exchange (82) - : +- HashAggregate (81) - : +- Scan parquet (80) - +- SortMergeJoin LeftSemi (105) - :- Sort (97) - : +- Exchange (96) - : +- Filter (95) - : +- Scan parquet (94) - +- Sort (104) - +- Project (103) - +- Filter (102) - +- HashAggregate (101) - +- Exchange (100) - +- HashAggregate (99) - +- Scan parquet (98) + TakeOrderedAndProject (103) + +- HashAggregate (102) + +- HashAggregate (101) + +- Project (100) + +- SortMergeJoin Inner (99) + :- Sort (86) + : +- Exchange (85) + : +- Project (84) + : +- SortMergeJoin Inner (83) + : :- Sort (68) + : : +- Exchange (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Sort (82) + : +- Exchange (81) + : +- SortMergeJoin LeftSemi (80) + : :- Sort (72) + : : +- Exchange (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- Sort (79) + : +- Project (78) + : +- Filter (77) + : +- HashAggregate (76) + : +- Exchange (75) + : +- HashAggregate (74) + : +- Scan parquet (73) + +- SortMergeJoin LeftSemi (98) + :- Sort (90) + : +- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Sort (97) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -133,462 +126,434 @@ Input [2]: [c_custkey#X, c_name#X] (8) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) SortExecTransformer -Input [2]: [c_custkey#X, c_name#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(16) InputAdapter -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(17) InputIteratorTransformer +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(18) SortExecTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(19) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(20) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(28) FilterExecTransformer +(26) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(29) ProjectExecTransformer +(27) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(30) SortExecTransformer -Input [1]: [l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(31) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(32) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(33) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(34) ColumnarExchange +(31) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(36) InputAdapter +(33) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(38) SortExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(39) ShuffledHashJoinExecTransformer +(35) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) ProjectExecTransformer +(36) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(41) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(42) ColumnarExchange +(38) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(39) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(44) InputAdapter +(40) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(45) InputIteratorTransformer +(41) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(46) SortExecTransformer -Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(47) Scan parquet +(42) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(48) NoopFilter +(43) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(49) ProjectExecTransformer +(44) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(50) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(51) ColumnarExchange +(46) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(47) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(53) InputAdapter +(48) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(54) InputIteratorTransformer +(49) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(55) SortExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(56) ReusedExchange [Reuses operator id: 23] +(50) ReusedExchange [Reuses operator id: 21] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(58) InputAdapter +(52) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(59) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(60) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(61) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(62) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(63) SortExecTransformer -Input [1]: [l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(64) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(65) ShuffledHashJoinExecTransformer +(58) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(66) ProjectExecTransformer +(59) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(67) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(68) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(69) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(70) TakeOrderedAndProjectExecTransformer +(63) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(71) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(72) Scan parquet +(65) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(73) Filter +(66) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(74) Exchange +(67) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) Sort +(68) Sort Input [2]: [c_custkey#X, c_name#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(76) Scan parquet +(69) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(77) Filter +(70) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(78) Exchange +(71) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Sort +(72) Sort Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(80) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(81) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(82) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(83) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(84) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(85) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(86) Sort +(79) Sort Input [1]: [l_orderkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(87) SortMergeJoin +(80) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(88) Exchange +(81) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Sort +(82) Sort Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(90) SortMergeJoin +(83) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(91) Project +(84) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(92) Exchange +(85) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(86) Sort Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(94) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(96) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(90) Sort Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(98) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(99) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(100) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(102) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(103) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(104) Sort +(97) Sort Input [1]: [l_orderkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(105) SortMergeJoin +(98) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(106) SortMergeJoin +(99) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(107) Project +(100) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(108) HashAggregate +(101) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(109) HashAggregate +(102) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(110) TakeOrderedAndProject +(103) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(111) AdaptiveSparkPlan +(104) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt index f6dc2eadff5b..4897d4c1072d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt @@ -1,39 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (24) - +- ^ RegularHashAggregateExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (37) - +- HashAggregate (36) - +- Project (35) - +- SortMergeJoin Inner (34) - :- Sort (29) - : +- Exchange (28) - : +- Project (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Sort (33) - +- Exchange (32) - +- Filter (31) - +- Scan parquet (30) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- SortMergeJoin Inner (32) + :- Sort (27) + : +- Exchange (26) + : +- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- Sort (31) + +- Exchange (30) + +- Filter (29) + +- Scan parquet (28) (1) Scan parquet @@ -69,142 +67,134 @@ Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] (8) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) SortExecTransformer -Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(18) SortExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(21) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(22) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(24) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(25) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(26) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(27) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(28) Exchange +(26) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Sort +(27) Sort Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(30) Scan parquet +(28) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(31) Filter +(29) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(32) Exchange +(30) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) Sort +(31) Sort Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(34) SortMergeJoin +(32) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(35) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(36) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(37) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(38) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt index f4c042c0ad53..2d4e189844cf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt @@ -1,130 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (146) +AdaptiveSparkPlan (136) +- == Final Plan == - VeloxColumnarToRowExec (96) - +- ^ SortExecTransformer (94) - +- ^ InputIteratorTransformer (93) - +- ShuffleQueryStage (91) - +- ColumnarExchange (90) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ SortExecTransformer (77) - : +- ^ InputIteratorTransformer (76) - : +- ShuffleQueryStage (74) - : +- ColumnarExchange (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (70) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ SortExecTransformer (35) - : : +- ^ InputIteratorTransformer (34) - : : +- ShuffleQueryStage (32) - : : +- ColumnarExchange (31) - : : +- ^ ProjectExecTransformer (29) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : : :- ^ SortExecTransformer (18) - : : : +- ^ InputIteratorTransformer (17) - : : : +- ShuffleQueryStage (15) - : : : +- ColumnarExchange (14) - : : : +- ^ ProjectExecTransformer (12) - : : : +- ^ NoopFilter (11) - : : : +- ^ Scan parquet (10) - : : +- ^ SortExecTransformer (27) - : : +- ^ InputIteratorTransformer (26) - : : +- ShuffleQueryStage (24) - : : +- ColumnarExchange (23) - : : +- ^ ProjectExecTransformer (21) - : : +- ^ NoopFilter (20) - : : +- ^ Scan parquet (19) - : +- ^ SortExecTransformer (61) - : +- ^ InputIteratorTransformer (60) - : +- ShuffleQueryStage (58) - : +- ColumnarExchange (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ FilterExecTransformer (54) - : +- ^ ProjectExecTransformer (53) - : +- ^ RegularHashAggregateExecTransformer (52) - : +- ^ RegularHashAggregateExecTransformer (51) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (50) - : :- ^ SortExecTransformer (44) - : : +- ^ InputIteratorTransformer (43) - : : +- ShuffleQueryStage (41) - : : +- ColumnarExchange (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ NoopFilter (37) - : : +- ^ Scan parquet (36) - : +- ^ SortExecTransformer (49) - : +- ^ InputIteratorTransformer (48) - : +- ShuffleQueryStage (46) - : +- ReusedExchange (45) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (86) + +- ^ SortExecTransformer (84) + +- ^ InputIteratorTransformer (83) + +- ShuffleQueryStage (81) + +- ColumnarExchange (80) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) + : :- ^ InputIteratorTransformer (31) + : : +- ShuffleQueryStage (29) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ShuffleQueryStage (14) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ NoopFilter (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ShuffleQueryStage (22) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ NoopFilter (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ NoopFilter (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41) + : +- ReusedExchange (40) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (145) - +- Exchange (144) - +- Project (143) - +- SortMergeJoin Inner (142) - :- Sort (136) - : +- Exchange (135) - : +- Project (134) - : +- SortMergeJoin LeftSemi (133) - : :- Sort (100) - : : +- Exchange (99) - : : +- Filter (98) - : : +- Scan parquet (97) - : +- Sort (132) - : +- Exchange (131) - : +- Project (130) - : +- SortMergeJoin Inner (129) - : :- Sort (112) - : : +- Exchange (111) - : : +- SortMergeJoin LeftSemi (110) - : : :- Sort (104) - : : : +- Exchange (103) - : : : +- Filter (102) - : : : +- Scan parquet (101) - : : +- Sort (109) - : : +- Exchange (108) - : : +- Project (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Sort (128) - : +- Exchange (127) - : +- Filter (126) - : +- HashAggregate (125) - : +- HashAggregate (124) - : +- SortMergeJoin LeftSemi (123) - : :- Sort (117) - : : +- Exchange (116) - : : +- Project (115) - : : +- Filter (114) - : : +- Scan parquet (113) - : +- Sort (122) - : +- Exchange (121) - : +- Project (120) - : +- Filter (119) - : +- Scan parquet (118) - +- Sort (141) - +- Exchange (140) - +- Project (139) - +- Filter (138) - +- Scan parquet (137) + Sort (135) + +- Exchange (134) + +- Project (133) + +- SortMergeJoin Inner (132) + :- Sort (126) + : +- Exchange (125) + : +- Project (124) + : +- SortMergeJoin LeftSemi (123) + : :- Sort (90) + : : +- Exchange (89) + : : +- Filter (88) + : : +- Scan parquet (87) + : +- Sort (122) + : +- Exchange (121) + : +- Project (120) + : +- SortMergeJoin Inner (119) + : :- Sort (102) + : : +- Exchange (101) + : : +- SortMergeJoin LeftSemi (100) + : : :- Sort (94) + : : : +- Exchange (93) + : : : +- Filter (92) + : : : +- Scan parquet (91) + : : +- Sort (99) + : : +- Exchange (98) + : : +- Project (97) + : : +- Filter (96) + : : +- Scan parquet (95) + : +- Sort (118) + : +- Exchange (117) + : +- Filter (116) + : +- HashAggregate (115) + : +- HashAggregate (114) + : +- SortMergeJoin LeftSemi (113) + : :- Sort (107) + : : +- Exchange (106) + : : +- Project (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Sort (112) + : +- Exchange (111) + : +- Project (110) + : +- Filter (109) + : +- Scan parquet (108) + +- Sort (131) + +- Exchange (130) + +- Project (129) + +- Filter (128) + +- Scan parquet (127) (1) Scan parquet @@ -160,584 +150,544 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] (8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) SortExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(16) InputAdapter -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(17) InputIteratorTransformer +(15) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(18) SortExecTransformer +(16) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 -(19) Scan parquet +(17) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(20) NoopFilter +(18) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [1]: [p_partkey#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [1]: [p_partkey#X] -(27) SortExecTransformer -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(28) ShuffledHashJoinExecTransformer +(25) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(35) SortExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 - -(36) Scan parquet +(32) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(37) NoopFilter +(33) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(38) ProjectExecTransformer +(34) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(39) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(40) ColumnarExchange +(36) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(41) ShuffleQueryStage +(37) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(42) InputAdapter +(38) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(43) InputIteratorTransformer +(39) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(44) SortExecTransformer -Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(45) ReusedExchange [Reuses operator id: 23] +(40) ReusedExchange [Reuses operator id: 21] Output [1]: [p_partkey#X] -(46) ShuffleQueryStage +(41) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(47) InputAdapter -Input [1]: [p_partkey#X] - -(48) InputIteratorTransformer +(42) InputAdapter Input [1]: [p_partkey#X] -(49) SortExecTransformer +(43) InputIteratorTransformer Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(50) ShuffledHashJoinExecTransformer +(44) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(51) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(53) ProjectExecTransformer +(47) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(54) FilterExecTransformer +(48) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(55) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(56) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(57) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(59) InputAdapter +(53) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(60) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(61) SortExecTransformer -Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 - -(62) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(67) InputAdapter +(60) InputAdapter Input [1]: [ps_suppkey#X] -(68) InputIteratorTransformer -Input [1]: [ps_suppkey#X] - -(69) SortExecTransformer +(61) InputIteratorTransformer Input [1]: [ps_suppkey#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(75) InputAdapter +(67) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(76) InputIteratorTransformer +(68) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(77) SortExecTransformer -Input [3]: [s_name#X, s_address#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(78) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(79) NoopFilter +(70) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(81) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(82) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(84) InputAdapter +(75) InputAdapter Input [1]: [n_nationkey#X] -(85) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [n_nationkey#X] -(86) SortExecTransformer -Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(87) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) ProjectExecTransformer +(78) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(89) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(90) ColumnarExchange +(80) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(91) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(92) InputAdapter +(82) InputAdapter Input [2]: [s_name#X, s_address#X] -(93) InputIteratorTransformer +(83) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(94) SortExecTransformer +(84) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(95) WholeStageCodegenTransformer (X) +(85) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(96) VeloxColumnarToRowExec +(86) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(97) Scan parquet +(87) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(98) Filter +(88) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(99) Exchange +(89) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Sort +(90) Sort Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(101) Scan parquet +(91) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(102) Filter +(92) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(103) Exchange +(93) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Sort +(94) Sort Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 -(105) Scan parquet +(95) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(106) Filter +(96) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(107) Project +(97) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(108) Exchange +(98) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Sort +(99) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(110) SortMergeJoin +(100) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(111) Exchange +(101) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Sort +(102) Sort Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 -(113) Scan parquet +(103) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(114) Filter +(104) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(115) Project +(105) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(116) Exchange +(106) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Sort +(107) Sort Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(118) Scan parquet +(108) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(119) Filter +(109) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(120) Project +(110) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(121) Exchange +(111) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(112) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(123) SortMergeJoin +(113) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(124) HashAggregate +(114) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(125) HashAggregate +(115) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(126) Filter +(116) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(127) Exchange +(117) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(118) Sort Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 -(129) SortMergeJoin +(119) SortMergeJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(130) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(131) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(122) Sort Input [1]: [ps_suppkey#X] Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 -(133) SortMergeJoin +(123) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(134) Project +(124) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(135) Exchange +(125) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Sort +(126) Sort Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(137) Scan parquet +(127) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(138) Filter +(128) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(139) Project +(129) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(140) Exchange +(130) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) Sort +(131) Sort Input [1]: [n_nationkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(142) SortMergeJoin +(132) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(143) Project +(133) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(144) Exchange +(134) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Sort +(135) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(146) AdaptiveSparkPlan +(136) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt index 8dec233a4cc7..46d95087ec90 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt @@ -1,122 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (137) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (91) - +- TakeOrderedAndProjectExecTransformer (90) - +- ^ RegularHashAggregateExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85) - +- ColumnarExchange (84) - +- ^ ProjectExecTransformer (82) - +- ^ FlushableHashAggregateExecTransformer (81) - +- ^ ProjectExecTransformer (80) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) - :- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ SortExecTransformer (52) - : : +- ^ InputIteratorTransformer (51) - : : +- ShuffleQueryStage (49) - : : +- ColumnarExchange (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) - : : :- ^ SortExecTransformer (9) - : : : +- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ SortExecTransformer (44) - : : +- ^ InputIteratorTransformer (43) - : : +- ShuffleQueryStage (41) - : : +- ColumnarExchange (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) - : : : :- ^ SortExecTransformer (18) - : : : : +- ^ InputIteratorTransformer (17) - : : : : +- ShuffleQueryStage (15) - : : : : +- ColumnarExchange (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ SortExecTransformer (26) - : : : +- ^ InputIteratorTransformer (25) - : : : +- ShuffleQueryStage (23) - : : : +- ColumnarExchange (22) - : : : +- ^ ProjectExecTransformer (20) - : : : +- ^ Scan parquet (19) - : : +- ^ SortExecTransformer (36) - : : +- ^ InputIteratorTransformer (35) - : : +- ShuffleQueryStage (33) - : : +- ColumnarExchange (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ NoopFilter (29) - : : +- ^ Scan parquet (28) - : +- ^ SortExecTransformer (61) - : +- ^ InputIteratorTransformer (60) - : +- ShuffleQueryStage (58) - : +- ColumnarExchange (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ NoopFilter (54) - : +- ^ Scan parquet (53) - +- ^ SortExecTransformer (78) - +- ^ InputIteratorTransformer (77) - +- ShuffleQueryStage (75) - +- ColumnarExchange (74) - +- ^ ProjectExecTransformer (72) - +- ^ NoopFilter (71) - +- ^ Scan parquet (70) + VeloxColumnarToRowExec (82) + +- TakeOrderedAndProjectExecTransformer (81) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + :- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) + : :- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ShuffleQueryStage (21) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ShuffleQueryStage (30) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == - TakeOrderedAndProject (136) - +- HashAggregate (135) - +- Exchange (134) - +- HashAggregate (133) - +- Project (132) - +- SortMergeJoin Inner (131) - :- Sort (125) - : +- Exchange (124) - : +- Project (123) - : +- SortMergeJoin Inner (122) - : :- Sort (116) - : : +- Exchange (115) - : : +- Project (114) - : : +- SortMergeJoin Inner (113) - : : :- Sort (95) - : : : +- Exchange (94) - : : : +- Filter (93) - : : : +- Scan parquet (92) - : : +- Sort (112) - : : +- Exchange (111) - : : +- SortMergeJoin LeftAnti (110) - : : :- SortMergeJoin LeftSemi (104) - : : : :- Sort (100) - : : : : +- Exchange (99) - : : : : +- Project (98) - : : : : +- Filter (97) - : : : : +- Scan parquet (96) - : : : +- Sort (103) - : : : +- Exchange (102) - : : : +- Scan parquet (101) - : : +- Sort (109) - : : +- Exchange (108) - : : +- Project (107) - : : +- Filter (106) - : : +- Scan parquet (105) - : +- Sort (121) - : +- Exchange (120) - : +- Project (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Sort (130) - +- Exchange (129) - +- Project (128) - +- Filter (127) - +- Scan parquet (126) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- SortMergeJoin Inner (122) + :- Sort (116) + : +- Exchange (115) + : +- Project (114) + : +- SortMergeJoin Inner (113) + : :- Sort (107) + : : +- Exchange (106) + : : +- Project (105) + : : +- SortMergeJoin Inner (104) + : : :- Sort (86) + : : : +- Exchange (85) + : : : +- Filter (84) + : : : +- Scan parquet (83) + : : +- Sort (103) + : : +- Exchange (102) + : : +- SortMergeJoin LeftAnti (101) + : : :- SortMergeJoin LeftSemi (95) + : : : :- Sort (91) + : : : : +- Exchange (90) + : : : : +- Project (89) + : : : : +- Filter (88) + : : : : +- Scan parquet (87) + : : : +- Sort (94) + : : : +- Exchange (93) + : : : +- Scan parquet (92) + : : +- Sort (100) + : : +- Exchange (99) + : : +- Project (98) + : : +- Filter (97) + : : +- Scan parquet (96) + : +- Sort (112) + : +- Exchange (111) + : +- Project (110) + : +- Filter (109) + : +- Scan parquet (108) + +- Sort (121) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -152,552 +143,516 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] (8) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) SortExecTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(18) SortExecTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(19) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(24) InputAdapter -Input [2]: [l_orderkey#X, l_suppkey#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(27) ShuffledHashJoinExecTransformer +(24) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(28) Scan parquet +(25) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(29) NoopFilter +(26) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(30) ProjectExecTransformer +(27) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(31) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(32) ColumnarExchange +(29) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(34) InputAdapter +(31) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(35) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(36) SortExecTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(37) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(38) ProjectExecTransformer +(34) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(40) ColumnarExchange +(36) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(41) ShuffleQueryStage +(37) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(42) InputAdapter +(38) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(43) InputIteratorTransformer +(39) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(44) SortExecTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 - -(45) ShuffledHashJoinExecTransformer +(40) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(50) InputAdapter -Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] - -(51) InputIteratorTransformer +(45) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(52) SortExecTransformer +(46) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(53) Scan parquet +(47) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(54) NoopFilter +(48) NoopFilter Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: [o_orderkey#X, o_orderstatus#X] -(55) ProjectExecTransformer +(49) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(56) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(57) ColumnarExchange +(51) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(52) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(59) InputAdapter +(53) InputAdapter Input [1]: [o_orderkey#X] -(60) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [o_orderkey#X] -(61) SortExecTransformer -Input [1]: [o_orderkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(62) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(60) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(68) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(69) SortExecTransformer -Input [2]: [s_name#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(70) Scan parquet +(62) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(71) NoopFilter +(63) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(72) ProjectExecTransformer +(64) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(73) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(74) ColumnarExchange +(66) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(75) ShuffleQueryStage +(67) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(76) InputAdapter -Input [1]: [n_nationkey#X] - -(77) InputIteratorTransformer +(68) InputAdapter Input [1]: [n_nationkey#X] -(78) SortExecTransformer +(69) InputIteratorTransformer Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(79) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(81) FlushableHashAggregateExecTransformer +(72) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(82) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(83) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(84) ColumnarExchange +(75) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(86) InputAdapter +(77) InputAdapter Input [2]: [s_name#X, count#X] -(87) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(88) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(89) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(90) TakeOrderedAndProjectExecTransformer +(81) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(91) VeloxColumnarToRowExec +(82) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(92) Scan parquet +(83) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(93) Filter +(84) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(94) Exchange +(85) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Sort +(86) Sort Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(96) Scan parquet +(87) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(97) Filter +(88) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(98) Project +(89) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(99) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Sort +(91) Sort Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(101) Scan parquet +(92) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(102) Exchange +(93) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Sort +(94) Sort Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(104) SortMergeJoin +(95) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(105) Scan parquet +(96) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(106) Filter +(97) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(107) Project +(98) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(108) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Sort +(100) Sort Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(110) SortMergeJoin +(101) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(111) Exchange +(102) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Sort +(103) Sort Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(113) SortMergeJoin +(104) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(114) Project +(105) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(115) Exchange +(106) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Sort +(107) Sort Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(117) Scan parquet +(108) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(109) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(119) Project +(110) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(120) Exchange +(111) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) Sort +(112) Sort Input [1]: [o_orderkey#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(122) SortMergeJoin +(113) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(123) Project +(114) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(124) Exchange +(115) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(116) Sort Input [2]: [s_name#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(126) Scan parquet +(117) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(127) Filter +(118) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(128) Project +(119) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(129) Exchange +(120) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Sort +(121) Sort Input [1]: [n_nationkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(131) SortMergeJoin +(122) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(132) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(133) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(134) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(135) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(136) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(137) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt index 27717fe5b2c1..c558da05763a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt @@ -1,47 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (48) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- SortMergeJoin LeftAnti (43) - :- Sort (39) - : +- Exchange (38) - : +- Filter (37) - : +- Scan parquet (36) - +- Sort (42) - +- Exchange (41) - +- Scan parquet (40) + Sort (47) + +- Exchange (46) + +- HashAggregate (45) + +- Exchange (44) + +- HashAggregate (43) + +- Project (42) + +- SortMergeJoin LeftAnti (41) + :- Sort (37) + : +- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Sort (40) + +- Exchange (39) + +- Scan parquet (38) (1) Scan parquet @@ -77,184 +75,176 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] (8) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) SortExecTransformer -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(12) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(12) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(13) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(15) InputAdapter +(14) InputAdapter Input [1]: [o_custkey#X] -(16) InputIteratorTransformer +(15) InputIteratorTransformer Input [1]: [o_custkey#X] -(17) SortExecTransformer -Input [1]: [o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(18) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(19) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(20) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(28) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(29) ColumnarExchange +(27) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(31) InputAdapter +(29) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(32) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) SortExecTransformer +(31) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(35) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(36) Scan parquet +(34) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(37) Filter +(35) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(38) Exchange +(36) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(37) Sort Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(40) Scan parquet +(38) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(39) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Sort +(40) Sort Input [1]: [o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(43) SortMergeJoin +(41) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(42) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(45) HashAggregate +(43) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(46) Exchange +(44) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(45) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(48) Exchange +(46) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(47) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(48) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt index 357abff126a8..41fa836a8680 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt @@ -1,64 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ RegularHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - :- ^ SortExecTransformer (26) - : +- ^ InputIteratorTransformer (25) - : +- ShuffleQueryStage (23) - : +- ColumnarExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ NoopFilter (11) - : +- ^ Scan parquet (10) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32) - +- ColumnarExchange (31) - +- ^ ProjectExecTransformer (29) - +- ^ NoopFilter (28) - +- ^ Scan parquet (27) + VeloxColumnarToRowExec (39) + +- TakeOrderedAndProjectExecTransformer (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + :- ^ InputIteratorTransformer (23) + : +- ShuffleQueryStage (21) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Project (63) - +- SortMergeJoin Inner (62) - :- Sort (56) - : +- Exchange (55) - : +- Project (54) - : +- SortMergeJoin Inner (53) - : :- Sort (48) - : : +- Exchange (47) - : : +- Project (46) - : : +- Filter (45) - : : +- Scan parquet (44) - : +- Sort (52) - : +- Exchange (51) - : +- Filter (50) - : +- Scan parquet (49) - +- Sort (61) - +- Exchange (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- SortMergeJoin Inner (58) + :- Sort (52) + : +- Exchange (51) + : +- Project (50) + : +- SortMergeJoin Inner (49) + : :- Sort (44) + : : +- Exchange (43) + : : +- Project (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Sort (48) + : +- Exchange (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Sort (57) + +- Exchange (56) + +- Project (55) + +- Filter (54) + +- Scan parquet (53) (1) Scan parquet @@ -94,262 +90,246 @@ Input [1]: [c_custkey#X] (8) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) SortExecTransformer -Input [1]: [c_custkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(18) SortExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(26) SortExecTransformer -Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(27) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(33) InputAdapter -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] - -(34) InputIteratorTransformer +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(35) SortExecTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(38) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(40) ProjectExecTransformer +(36) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(41) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(38) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(43) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(44) Scan parquet +(40) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(45) Filter +(41) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(46) Project +(42) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(47) Exchange +(43) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(44) Sort Input [1]: [c_custkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(49) Scan parquet +(45) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(50) Filter +(46) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(51) Exchange +(47) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(48) Sort Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(53) SortMergeJoin +(49) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(54) Project +(50) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(55) Exchange +(51) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Sort +(52) Sort Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(57) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(58) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(59) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(60) Exchange +(56) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(57) Sort Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(62) SortMergeJoin +(58) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(63) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(64) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(65) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(66) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(67) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt index 65912d42eb38..6c17ed524036 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt @@ -1,51 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (53) - +- Exchange (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- SortMergeJoin LeftSemi (47) - :- Sort (41) - : +- Exchange (40) - : +- Project (39) - : +- Filter (38) - : +- Scan parquet (37) - +- Sort (46) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftSemi (45) + :- Sort (39) + : +- Exchange (38) + : +- Project (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Sort (44) + +- Exchange (43) + +- Project (42) + +- Filter (41) + +- Scan parquet (40) (1) Scan parquet @@ -81,202 +79,194 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] (8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) SortExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [1]: [l_orderkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [l_orderkey#X] -(18) SortExecTransformer -Input [1]: [l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(21) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(24) ColumnarExchange +(22) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(23) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(26) InputAdapter +(24) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(27) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(28) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(29) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) ColumnarExchange +(28) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(32) InputAdapter +(30) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(33) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(34) SortExecTransformer +(32) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(36) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(37) Scan parquet +(35) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(38) Filter +(36) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(39) Project +(37) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(40) Exchange +(38) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(39) Sort Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(42) Scan parquet +(40) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(43) Filter +(41) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(44) Project +(42) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(45) Exchange +(43) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(44) Sort Input [1]: [l_orderkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(47) SortMergeJoin +(45) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(46) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(49) HashAggregate +(47) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(50) Exchange +(48) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(49) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(52) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) Sort +(51) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(54) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt index f6a416a9f449..05512dbd5f93 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt @@ -1,135 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (154) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (104) - +- ^ SortExecTransformer (102) - +- ^ InputIteratorTransformer (101) - +- ShuffleQueryStage (99) - +- ColumnarExchange (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ SortExecTransformer (77) - : +- ^ InputIteratorTransformer (76) - : +- ShuffleQueryStage (74) - : +- ColumnarExchange (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ SortExecTransformer (60) - : : +- ^ InputIteratorTransformer (59) - : : +- ShuffleQueryStage (57) - : : +- ColumnarExchange (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ SortExecTransformer (43) - : : : +- ^ InputIteratorTransformer (42) - : : : +- ShuffleQueryStage (40) - : : : +- ColumnarExchange (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : : : :- ^ SortExecTransformer (26) - : : : : +- ^ InputIteratorTransformer (25) - : : : : +- ShuffleQueryStage (23) - : : : : +- ColumnarExchange (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ SortExecTransformer (9) - : : : : : +- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ SortExecTransformer (18) - : : : : +- ^ InputIteratorTransformer (17) - : : : : +- ShuffleQueryStage (15) - : : : : +- ColumnarExchange (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ SortExecTransformer (35) - : : : +- ^ InputIteratorTransformer (34) - : : : +- ShuffleQueryStage (32) - : : : +- ColumnarExchange (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ SortExecTransformer (52) - : : +- ^ InputIteratorTransformer (51) - : : +- ShuffleQueryStage (49) - : : +- ColumnarExchange (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (153) - +- Exchange (152) - +- HashAggregate (151) - +- Exchange (150) - +- HashAggregate (149) - +- Project (148) - +- SortMergeJoin Inner (147) - :- Sort (141) - : +- Exchange (140) - : +- Project (139) - : +- SortMergeJoin Inner (138) - : :- Sort (133) - : : +- Exchange (132) - : : +- Project (131) - : : +- SortMergeJoin Inner (130) - : : :- Sort (125) - : : : +- Exchange (124) - : : : +- Project (123) - : : : +- SortMergeJoin Inner (122) - : : : :- Sort (117) - : : : : +- Exchange (116) - : : : : +- Project (115) - : : : : +- SortMergeJoin Inner (114) - : : : : :- Sort (108) - : : : : : +- Exchange (107) - : : : : : +- Filter (106) - : : : : : +- Scan parquet (105) - : : : : +- Sort (113) - : : : : +- Exchange (112) - : : : : +- Project (111) - : : : : +- Filter (110) - : : : : +- Scan parquet (109) - : : : +- Sort (121) - : : : +- Exchange (120) - : : : +- Filter (119) - : : : +- Scan parquet (118) - : : +- Sort (129) - : : +- Exchange (128) - : : +- Filter (127) - : : +- Scan parquet (126) - : +- Sort (137) - : +- Exchange (136) - : +- Filter (135) - : +- Scan parquet (134) - +- Sort (146) - +- Exchange (145) - +- Project (144) - +- Filter (143) - +- Scan parquet (142) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- SortMergeJoin Inner (137) + :- Sort (131) + : +- Exchange (130) + : +- Project (129) + : +- SortMergeJoin Inner (128) + : :- Sort (123) + : : +- Exchange (122) + : : +- Project (121) + : : +- SortMergeJoin Inner (120) + : : :- Sort (115) + : : : +- Exchange (114) + : : : +- Project (113) + : : : +- SortMergeJoin Inner (112) + : : : :- Sort (107) + : : : : +- Exchange (106) + : : : : +- Project (105) + : : : : +- SortMergeJoin Inner (104) + : : : : :- Sort (98) + : : : : : +- Exchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Sort (103) + : : : : +- Exchange (102) + : : : : +- Project (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- Sort (111) + : : : +- Exchange (110) + : : : +- Filter (109) + : : : +- Scan parquet (108) + : : +- Sort (119) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Sort (127) + : +- Exchange (126) + : +- Filter (125) + : +- Scan parquet (124) + +- Sort (136) + +- Exchange (135) + +- Project (134) + +- Filter (133) + +- Scan parquet (132) (1) Scan parquet @@ -165,618 +155,578 @@ Input [2]: [c_custkey#X, c_nationkey#X] (8) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) SortExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(18) SortExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(24) InputAdapter -Input [2]: [c_nationkey#X, o_orderkey#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(27) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(35) SortExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(41) InputAdapter -Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(42) InputIteratorTransformer +(37) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(43) SortExecTransformer +(38) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 -(44) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(50) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(51) InputIteratorTransformer +(45) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(52) SortExecTransformer +(46) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(55) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(56) ColumnarExchange +(50) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(58) InputAdapter +(52) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(59) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(60) SortExecTransformer -Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(61) Scan parquet +(54) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(62) NoopFilter +(55) NoopFilter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(60) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] - -(69) SortExecTransformer +(61) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(75) InputAdapter -Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] - -(76) InputIteratorTransformer +(67) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(77) SortExecTransformer +(68) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 -(78) Scan parquet +(69) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(79) NoopFilter +(70) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(81) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(82) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(84) InputAdapter -Input [1]: [r_regionkey#X] - -(85) InputIteratorTransformer +(75) InputAdapter Input [1]: [r_regionkey#X] -(86) SortExecTransformer +(76) InputIteratorTransformer Input [1]: [r_regionkey#X] -Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 -(87) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(88) ProjectExecTransformer +(78) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(89) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(90) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(91) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(82) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(83) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(84) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(85) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(97) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(98) ColumnarExchange +(88) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(99) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(100) InputAdapter +(90) InputAdapter Input [2]: [n_name#X, revenue#X] -(101) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(102) SortExecTransformer +(92) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(103) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(104) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(105) Scan parquet +(95) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(106) Filter +(96) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(107) Exchange +(97) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Sort +(98) Sort Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(109) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(110) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(111) Project +(101) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(112) Exchange +(102) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Sort +(103) Sort Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(114) SortMergeJoin +(104) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(115) Project +(105) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(106) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Sort +(107) Sort Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(118) Scan parquet +(108) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(119) Filter +(109) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(120) Exchange +(110) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) Sort +(111) Sort Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(122) SortMergeJoin +(112) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(123) Project +(113) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(124) Exchange +(114) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(115) Sort Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 -(126) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(127) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(128) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(119) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 -(130) SortMergeJoin +(120) SortMergeJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(131) Project +(121) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(132) Exchange +(122) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(123) Sort Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(134) Scan parquet +(124) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(135) Filter +(125) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(136) Exchange +(126) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) Sort +(127) Sort Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(138) SortMergeJoin +(128) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(139) Project +(129) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(140) Exchange +(130) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) Sort +(131) Sort Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 -(142) Scan parquet +(132) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(143) Filter +(133) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(144) Project +(134) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(145) Exchange +(135) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Sort +(136) Sort Input [1]: [r_regionkey#X] Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 -(147) SortMergeJoin +(137) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(148) Project +(138) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(149) HashAggregate +(139) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(150) Exchange +(140) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) HashAggregate +(141) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(152) Exchange +(142) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(153) Sort +(143) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(154) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt index 68ba018974e0..d2bc4697bbe6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt @@ -1,130 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (148) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (100) - +- ^ SortExecTransformer (98) - +- ^ InputIteratorTransformer (97) - +- ShuffleQueryStage (95) - +- ColumnarExchange (94) - +- ^ RegularHashAggregateExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ ProjectExecTransformer (86) - +- ^ FlushableHashAggregateExecTransformer (85) - +- ^ ProjectExecTransformer (84) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (83) - :- ^ SortExecTransformer (77) - : +- ^ InputIteratorTransformer (76) - : +- ShuffleQueryStage (74) - : +- ColumnarExchange (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ SortExecTransformer (60) - : : +- ^ InputIteratorTransformer (59) - : : +- ShuffleQueryStage (57) - : : +- ColumnarExchange (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ SortExecTransformer (43) - : : : +- ^ InputIteratorTransformer (42) - : : : +- ShuffleQueryStage (40) - : : : +- ColumnarExchange (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ SortExecTransformer (26) - : : : : +- ^ InputIteratorTransformer (25) - : : : : +- ShuffleQueryStage (23) - : : : : +- ColumnarExchange (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ SortExecTransformer (9) - : : : : : +- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ SortExecTransformer (18) - : : : : +- ^ InputIteratorTransformer (17) - : : : : +- ShuffleQueryStage (15) - : : : : +- ColumnarExchange (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ SortExecTransformer (35) - : : : +- ^ InputIteratorTransformer (34) - : : : +- ShuffleQueryStage (32) - : : : +- ColumnarExchange (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ SortExecTransformer (52) - : : +- ^ InputIteratorTransformer (51) - : : +- ShuffleQueryStage (49) - : : +- ColumnarExchange (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ReusedExchange (78) + VeloxColumnarToRowExec (90) + +- ^ SortExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ShuffleQueryStage (70) + +- ReusedExchange (69) +- == Initial Plan == - Sort (147) - +- Exchange (146) - +- HashAggregate (145) - +- Exchange (144) - +- HashAggregate (143) - +- Project (142) - +- SortMergeJoin Inner (141) - :- Sort (136) - : +- Exchange (135) - : +- Project (134) - : +- SortMergeJoin Inner (133) - : :- Sort (128) - : : +- Exchange (127) - : : +- Project (126) - : : +- SortMergeJoin Inner (125) - : : :- Sort (120) - : : : +- Exchange (119) - : : : +- Project (118) - : : : +- SortMergeJoin Inner (117) - : : : :- Sort (112) - : : : : +- Exchange (111) - : : : : +- Project (110) - : : : : +- SortMergeJoin Inner (109) - : : : : :- Sort (104) - : : : : : +- Exchange (103) - : : : : : +- Filter (102) - : : : : : +- Scan parquet (101) - : : : : +- Sort (108) - : : : : +- Exchange (107) - : : : : +- Filter (106) - : : : : +- Scan parquet (105) - : : : +- Sort (116) - : : : +- Exchange (115) - : : : +- Filter (114) - : : : +- Scan parquet (113) - : : +- Sort (124) - : : +- Exchange (123) - : : +- Filter (122) - : : +- Scan parquet (121) - : +- Sort (132) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Sort (140) - +- Exchange (139) - +- Filter (138) - +- Scan parquet (137) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Sort (126) + : +- Exchange (125) + : +- Project (124) + : +- SortMergeJoin Inner (123) + : :- Sort (118) + : : +- Exchange (117) + : : +- Project (116) + : : +- SortMergeJoin Inner (115) + : : :- Sort (110) + : : : +- Exchange (109) + : : : +- Project (108) + : : : +- SortMergeJoin Inner (107) + : : : :- Sort (102) + : : : : +- Exchange (101) + : : : : +- Project (100) + : : : : +- SortMergeJoin Inner (99) + : : : : :- Sort (94) + : : : : : +- Exchange (93) + : : : : : +- Filter (92) + : : : : : +- Scan parquet (91) + : : : : +- Sort (98) + : : : : +- Exchange (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Sort (106) + : : : +- Exchange (105) + : : : +- Filter (104) + : : : +- Scan parquet (103) + : : +- Sort (114) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Sort (122) + : +- Exchange (121) + : +- Filter (120) + : +- Scan parquet (119) + +- Sort (130) + +- Exchange (129) + +- Filter (128) + +- Scan parquet (127) (1) Scan parquet @@ -160,590 +150,550 @@ Input [2]: [s_suppkey#X, s_nationkey#X] (8) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) SortExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(18) SortExecTransformer -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) SortExecTransformer -Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(27) Scan parquet +(24) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X, o_custkey#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(35) SortExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(41) InputAdapter +(37) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(42) InputIteratorTransformer +(38) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(43) SortExecTransformer -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(44) Scan parquet +(39) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(50) InputAdapter +(45) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(51) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(52) SortExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(55) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(56) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(58) InputAdapter +(52) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(59) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(60) SortExecTransformer -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(61) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(62) NoopFilter +(55) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(68) InputIteratorTransformer +(60) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(69) SortExecTransformer +(61) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter -Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] - -(76) InputIteratorTransformer +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(77) SortExecTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(78) ReusedExchange [Reuses operator id: 65] +(69) ReusedExchange [Reuses operator id: 58] Output [2]: [n_nationkey#X, n_name#X] -(79) ShuffleQueryStage +(70) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(80) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(81) InputIteratorTransformer +(71) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(82) SortExecTransformer +(72) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(83) ShuffledHashJoinExecTransformer +(73) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(84) ProjectExecTransformer +(74) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(85) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(86) ProjectExecTransformer +(76) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(87) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(88) ColumnarExchange +(78) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(79) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(90) InputAdapter +(80) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(91) InputIteratorTransformer +(81) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(92) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(93) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(94) ColumnarExchange +(84) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(95) ShuffleQueryStage +(85) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(96) InputAdapter +(86) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(97) InputIteratorTransformer +(87) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(98) SortExecTransformer +(88) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(99) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(100) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(101) Scan parquet +(91) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(102) Filter +(92) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(103) Exchange +(93) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Sort +(94) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(105) Scan parquet +(95) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(106) Filter +(96) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(107) Exchange +(97) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Sort +(98) Sort Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(109) SortMergeJoin +(99) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(110) Project +(100) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(111) Exchange +(101) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Sort +(102) Sort Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(113) Scan parquet +(103) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(114) Filter +(104) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(115) Exchange +(105) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Sort +(106) Sort Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(117) SortMergeJoin +(107) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(118) Project +(108) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(119) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(110) Sort Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(121) Scan parquet +(111) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(122) Filter +(112) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(123) Exchange +(113) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) Sort +(114) Sort Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(125) SortMergeJoin +(115) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(126) Project +(116) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(127) Exchange +(117) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(118) Sort Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(129) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(130) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(131) Exchange +(121) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(122) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(133) SortMergeJoin +(123) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(134) Project +(124) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(135) Exchange +(125) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Sort +(126) Sort Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(137) Scan parquet +(127) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(138) Filter +(128) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(139) Exchange +(129) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(140) Sort +(130) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(141) SortMergeJoin +(131) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(142) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(143) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(144) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(146) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(147) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(148) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt index 5d3be5d8c3bc..6ccdc78c116e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt @@ -1,178 +1,164 @@ == Physical Plan == -AdaptiveSparkPlan (205) +AdaptiveSparkPlan (191) +- == Final Plan == - VeloxColumnarToRowExec (139) - +- ^ SortExecTransformer (137) - +- ^ InputIteratorTransformer (136) - +- ShuffleQueryStage (134) - +- ColumnarExchange (133) - +- ^ ProjectExecTransformer (131) - +- ^ RegularHashAggregateExecTransformer (130) - +- ^ InputIteratorTransformer (129) - +- ShuffleQueryStage (127) - +- ColumnarExchange (126) - +- ^ ProjectExecTransformer (124) - +- ^ FlushableHashAggregateExecTransformer (123) - +- ^ ProjectExecTransformer (122) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) - :- ^ SortExecTransformer (111) - : +- ^ InputIteratorTransformer (110) - : +- ShuffleQueryStage (108) - : +- ColumnarExchange (107) - : +- ^ ProjectExecTransformer (105) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - : :- ^ SortExecTransformer (94) - : : +- ^ InputIteratorTransformer (93) - : : +- ShuffleQueryStage (91) - : : +- ColumnarExchange (90) - : : +- ^ ProjectExecTransformer (88) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - : : :- ^ SortExecTransformer (77) - : : : +- ^ InputIteratorTransformer (76) - : : : +- ShuffleQueryStage (74) - : : : +- ColumnarExchange (73) - : : : +- ^ ProjectExecTransformer (71) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : : : :- ^ SortExecTransformer (60) - : : : : +- ^ InputIteratorTransformer (59) - : : : : +- ShuffleQueryStage (57) - : : : : +- ColumnarExchange (56) - : : : : +- ^ ProjectExecTransformer (54) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : : : :- ^ SortExecTransformer (43) - : : : : : +- ^ InputIteratorTransformer (42) - : : : : : +- ShuffleQueryStage (40) - : : : : : +- ColumnarExchange (39) - : : : : : +- ^ ProjectExecTransformer (37) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : : : :- ^ SortExecTransformer (26) - : : : : : : +- ^ InputIteratorTransformer (25) - : : : : : : +- ShuffleQueryStage (23) - : : : : : : +- ColumnarExchange (22) - : : : : : : +- ^ ProjectExecTransformer (20) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : : : :- ^ SortExecTransformer (9) - : : : : : : : +- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ SortExecTransformer (18) - : : : : : : +- ^ InputIteratorTransformer (17) - : : : : : : +- ShuffleQueryStage (15) - : : : : : : +- ColumnarExchange (14) - : : : : : : +- ^ ProjectExecTransformer (12) - : : : : : : +- ^ NoopFilter (11) - : : : : : : +- ^ Scan parquet (10) - : : : : : +- ^ SortExecTransformer (35) - : : : : : +- ^ InputIteratorTransformer (34) - : : : : : +- ShuffleQueryStage (32) - : : : : : +- ColumnarExchange (31) - : : : : : +- ^ ProjectExecTransformer (29) - : : : : : +- ^ NoopFilter (28) - : : : : : +- ^ Scan parquet (27) - : : : : +- ^ SortExecTransformer (52) - : : : : +- ^ InputIteratorTransformer (51) - : : : : +- ShuffleQueryStage (49) - : : : : +- ColumnarExchange (48) - : : : : +- ^ ProjectExecTransformer (46) - : : : : +- ^ NoopFilter (45) - : : : : +- ^ Scan parquet (44) - : : : +- ^ SortExecTransformer (69) - : : : +- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ NoopFilter (62) - : : : +- ^ Scan parquet (61) - : : +- ^ SortExecTransformer (86) - : : +- ^ InputIteratorTransformer (85) - : : +- ShuffleQueryStage (83) - : : +- ColumnarExchange (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ NoopFilter (79) - : : +- ^ Scan parquet (78) - : +- ^ SortExecTransformer (103) - : +- ^ InputIteratorTransformer (102) - : +- ShuffleQueryStage (100) - : +- ColumnarExchange (99) - : +- ^ ProjectExecTransformer (97) - : +- ^ NoopFilter (96) - : +- ^ Scan parquet (95) - +- ^ SortExecTransformer (120) - +- ^ InputIteratorTransformer (119) - +- ShuffleQueryStage (117) - +- ColumnarExchange (116) - +- ^ ProjectExecTransformer (114) - +- ^ NoopFilter (113) - +- ^ Scan parquet (112) + VeloxColumnarToRowExec (125) + +- ^ SortExecTransformer (123) + +- ^ InputIteratorTransformer (122) + +- ShuffleQueryStage (120) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ShuffleQueryStage (113) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (83) + : : +- ShuffleQueryStage (81) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ShuffleQueryStage (66) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ShuffleQueryStage (51) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ShuffleQueryStage (36) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ShuffleQueryStage (21) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ShuffleQueryStage (6) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ShuffleQueryStage (14) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ShuffleQueryStage (29) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ NoopFilter (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ShuffleQueryStage (44) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ NoopFilter (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ShuffleQueryStage (59) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ NoopFilter (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ShuffleQueryStage (74) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ NoopFilter (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ NoopFilter (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ShuffleQueryStage (104) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ NoopFilter (100) + +- ^ Scan parquet (99) +- == Initial Plan == - Sort (204) - +- Exchange (203) - +- HashAggregate (202) - +- Exchange (201) - +- HashAggregate (200) - +- Project (199) - +- SortMergeJoin Inner (198) - :- Sort (192) - : +- Exchange (191) - : +- Project (190) - : +- SortMergeJoin Inner (189) - : :- Sort (184) - : : +- Exchange (183) - : : +- Project (182) - : : +- SortMergeJoin Inner (181) - : : :- Sort (176) - : : : +- Exchange (175) - : : : +- Project (174) - : : : +- SortMergeJoin Inner (173) - : : : :- Sort (168) - : : : : +- Exchange (167) - : : : : +- Project (166) - : : : : +- SortMergeJoin Inner (165) - : : : : :- Sort (160) - : : : : : +- Exchange (159) - : : : : : +- Project (158) - : : : : : +- SortMergeJoin Inner (157) - : : : : : :- Sort (152) - : : : : : : +- Exchange (151) - : : : : : : +- Project (150) - : : : : : : +- SortMergeJoin Inner (149) - : : : : : : :- Sort (144) - : : : : : : : +- Exchange (143) - : : : : : : : +- Project (142) - : : : : : : : +- Filter (141) - : : : : : : : +- Scan parquet (140) - : : : : : : +- Sort (148) - : : : : : : +- Exchange (147) - : : : : : : +- Filter (146) - : : : : : : +- Scan parquet (145) - : : : : : +- Sort (156) - : : : : : +- Exchange (155) - : : : : : +- Filter (154) - : : : : : +- Scan parquet (153) - : : : : +- Sort (164) - : : : : +- Exchange (163) - : : : : +- Filter (162) - : : : : +- Scan parquet (161) - : : : +- Sort (172) - : : : +- Exchange (171) - : : : +- Filter (170) - : : : +- Scan parquet (169) - : : +- Sort (180) - : : +- Exchange (179) - : : +- Filter (178) - : : +- Scan parquet (177) - : +- Sort (188) - : +- Exchange (187) - : +- Filter (186) - : +- Scan parquet (185) - +- Sort (197) - +- Exchange (196) - +- Project (195) - +- Filter (194) - +- Scan parquet (193) + Sort (190) + +- Exchange (189) + +- HashAggregate (188) + +- Exchange (187) + +- HashAggregate (186) + +- Project (185) + +- SortMergeJoin Inner (184) + :- Sort (178) + : +- Exchange (177) + : +- Project (176) + : +- SortMergeJoin Inner (175) + : :- Sort (170) + : : +- Exchange (169) + : : +- Project (168) + : : +- SortMergeJoin Inner (167) + : : :- Sort (162) + : : : +- Exchange (161) + : : : +- Project (160) + : : : +- SortMergeJoin Inner (159) + : : : :- Sort (154) + : : : : +- Exchange (153) + : : : : +- Project (152) + : : : : +- SortMergeJoin Inner (151) + : : : : :- Sort (146) + : : : : : +- Exchange (145) + : : : : : +- Project (144) + : : : : : +- SortMergeJoin Inner (143) + : : : : : :- Sort (138) + : : : : : : +- Exchange (137) + : : : : : : +- Project (136) + : : : : : : +- SortMergeJoin Inner (135) + : : : : : : :- Sort (130) + : : : : : : : +- Exchange (129) + : : : : : : : +- Project (128) + : : : : : : : +- Filter (127) + : : : : : : : +- Scan parquet (126) + : : : : : : +- Sort (134) + : : : : : : +- Exchange (133) + : : : : : : +- Filter (132) + : : : : : : +- Scan parquet (131) + : : : : : +- Sort (142) + : : : : : +- Exchange (141) + : : : : : +- Filter (140) + : : : : : +- Scan parquet (139) + : : : : +- Sort (150) + : : : : +- Exchange (149) + : : : : +- Filter (148) + : : : : +- Scan parquet (147) + : : : +- Sort (158) + : : : +- Exchange (157) + : : : +- Filter (156) + : : : +- Scan parquet (155) + : : +- Sort (166) + : : +- Exchange (165) + : : +- Filter (164) + : : +- Scan parquet (163) + : +- Sort (174) + : +- Exchange (173) + : +- Filter (172) + : +- Scan parquet (171) + +- Sort (183) + +- Exchange (182) + +- Project (181) + +- Filter (180) + +- Scan parquet (179) (1) Scan parquet @@ -208,830 +194,774 @@ Input [1]: [p_partkey#X] (8) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) SortExecTransformer -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(18) SortExecTransformer -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(24) InputAdapter -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(27) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(35) SortExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(41) InputAdapter -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(42) InputIteratorTransformer +(37) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(43) SortExecTransformer +(38) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(44) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(50) InputAdapter -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(51) InputIteratorTransformer +(45) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(52) SortExecTransformer +(46) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(55) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(56) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(58) InputAdapter +(52) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(59) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(60) SortExecTransformer -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(61) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) NoopFilter +(55) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(60) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(68) InputIteratorTransformer -Input [2]: [c_custkey#X, c_nationkey#X] - -(69) SortExecTransformer +(61) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(75) InputAdapter -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] - -(76) InputIteratorTransformer +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(77) SortExecTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(78) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(79) NoopFilter +(70) NoopFilter Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X, n_regionkey#X] -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(81) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(82) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(84) InputAdapter -Input [2]: [n_nationkey#X, n_regionkey#X] - -(85) InputIteratorTransformer +(75) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(86) SortExecTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(87) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) ProjectExecTransformer +(78) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(89) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(90) ColumnarExchange +(80) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(91) ShuffleQueryStage +(81) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(92) InputAdapter +(82) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(93) InputIteratorTransformer +(83) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(94) SortExecTransformer -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(95) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(96) NoopFilter +(85) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(97) ProjectExecTransformer +(86) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(98) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(99) ColumnarExchange +(88) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(101) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(102) InputIteratorTransformer +(90) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(103) SortExecTransformer +(91) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(104) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(105) ProjectExecTransformer +(93) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(106) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(107) ColumnarExchange +(95) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(108) ShuffleQueryStage +(96) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(109) InputAdapter +(97) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(110) InputIteratorTransformer +(98) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(111) SortExecTransformer -Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 - -(112) Scan parquet +(99) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(113) NoopFilter +(100) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(114) ProjectExecTransformer +(101) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(115) WholeStageCodegenTransformer (X) +(102) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(116) ColumnarExchange +(103) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(117) ShuffleQueryStage +(104) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(118) InputAdapter -Input [1]: [r_regionkey#X] - -(119) InputIteratorTransformer +(105) InputAdapter Input [1]: [r_regionkey#X] -(120) SortExecTransformer +(106) InputIteratorTransformer Input [1]: [r_regionkey#X] -Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 -(121) ShuffledHashJoinExecTransformer +(107) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(122) ProjectExecTransformer +(108) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(123) FlushableHashAggregateExecTransformer +(109) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(124) ProjectExecTransformer +(110) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) WholeStageCodegenTransformer (X) +(111) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(126) ColumnarExchange +(112) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(127) ShuffleQueryStage +(113) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(128) InputAdapter +(114) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(129) InputIteratorTransformer +(115) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(130) RegularHashAggregateExecTransformer +(116) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(131) ProjectExecTransformer +(117) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(132) WholeStageCodegenTransformer (X) +(118) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(133) ColumnarExchange +(119) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(134) ShuffleQueryStage +(120) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(135) InputAdapter +(121) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(136) InputIteratorTransformer +(122) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(137) SortExecTransformer +(123) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(138) WholeStageCodegenTransformer (X) +(124) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(139) VeloxColumnarToRowExec +(125) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(140) Scan parquet +(126) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(141) Filter +(127) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(142) Project +(128) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(143) Exchange +(129) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(130) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(145) Scan parquet +(131) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(146) Filter +(132) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(147) Exchange +(133) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Sort +(134) Sort Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(149) SortMergeJoin +(135) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(150) Project +(136) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(151) Exchange +(137) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Sort +(138) Sort Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(153) Scan parquet +(139) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(154) Filter +(140) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(155) Exchange +(141) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(156) Sort +(142) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(157) SortMergeJoin +(143) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(158) Project +(144) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(159) Exchange +(145) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Sort +(146) Sort Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(161) Scan parquet +(147) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(162) Filter +(148) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(163) Exchange +(149) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(164) Sort +(150) Sort Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(165) SortMergeJoin +(151) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(166) Project +(152) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(167) Exchange +(153) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(168) Sort +(154) Sort Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(169) Scan parquet +(155) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(170) Filter +(156) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(171) Exchange +(157) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(172) Sort +(158) Sort Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(173) SortMergeJoin +(159) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(174) Project +(160) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(175) Exchange +(161) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(162) Sort Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(177) Scan parquet +(163) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(178) Filter +(164) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(179) Exchange +(165) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(180) Sort +(166) Sort Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(181) SortMergeJoin +(167) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(182) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(183) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(184) Sort +(170) Sort Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(185) Scan parquet +(171) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(186) Filter +(172) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(187) Exchange +(173) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(188) Sort +(174) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(189) SortMergeJoin +(175) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(190) Project +(176) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(191) Exchange +(177) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(192) Sort +(178) Sort Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 -(193) Scan parquet +(179) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(194) Filter +(180) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(195) Project +(181) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(196) Exchange +(182) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(197) Sort +(183) Sort Input [1]: [r_regionkey#X] Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 -(198) SortMergeJoin +(184) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(199) Project +(185) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(200) HashAggregate +(186) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(201) Exchange +(187) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(202) HashAggregate +(188) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(203) Exchange +(189) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(204) Sort +(190) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(205) AdaptiveSparkPlan +(191) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt index 8d5ea0ba770e..51699c3877d3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt @@ -1,134 +1,124 @@ == Physical Plan == -AdaptiveSparkPlan (153) +AdaptiveSparkPlan (143) +- == Final Plan == - VeloxColumnarToRowExec (104) - +- ^ SortExecTransformer (102) - +- ^ InputIteratorTransformer (101) - +- ShuffleQueryStage (99) - +- ColumnarExchange (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ SortExecTransformer (77) - : +- ^ InputIteratorTransformer (76) - : +- ShuffleQueryStage (74) - : +- ColumnarExchange (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : :- ^ SortExecTransformer (60) - : : +- ^ InputIteratorTransformer (59) - : : +- ShuffleQueryStage (57) - : : +- ColumnarExchange (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : :- ^ SortExecTransformer (43) - : : : +- ^ InputIteratorTransformer (42) - : : : +- ShuffleQueryStage (40) - : : : +- ColumnarExchange (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ SortExecTransformer (26) - : : : : +- ^ InputIteratorTransformer (25) - : : : : +- ShuffleQueryStage (23) - : : : : +- ColumnarExchange (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ SortExecTransformer (9) - : : : : : +- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ SortExecTransformer (18) - : : : : +- ^ InputIteratorTransformer (17) - : : : : +- ShuffleQueryStage (15) - : : : : +- ColumnarExchange (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ SortExecTransformer (35) - : : : +- ^ InputIteratorTransformer (34) - : : : +- ShuffleQueryStage (32) - : : : +- ColumnarExchange (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ SortExecTransformer (52) - : : +- ^ InputIteratorTransformer (51) - : : +- ShuffleQueryStage (49) - : : +- ColumnarExchange (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (152) - +- Exchange (151) - +- HashAggregate (150) - +- Exchange (149) - +- HashAggregate (148) - +- Project (147) - +- SortMergeJoin Inner (146) - :- Sort (141) - : +- Exchange (140) - : +- Project (139) - : +- SortMergeJoin Inner (138) - : :- Sort (133) - : : +- Exchange (132) - : : +- Project (131) - : : +- SortMergeJoin Inner (130) - : : :- Sort (125) - : : : +- Exchange (124) - : : : +- Project (123) - : : : +- SortMergeJoin Inner (122) - : : : :- Sort (117) - : : : : +- Exchange (116) - : : : : +- Project (115) - : : : : +- SortMergeJoin Inner (114) - : : : : :- Sort (109) - : : : : : +- Exchange (108) - : : : : : +- Project (107) - : : : : : +- Filter (106) - : : : : : +- Scan parquet (105) - : : : : +- Sort (113) - : : : : +- Exchange (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Sort (121) - : : : +- Exchange (120) - : : : +- Filter (119) - : : : +- Scan parquet (118) - : : +- Sort (129) - : : +- Exchange (128) - : : +- Filter (127) - : : +- Scan parquet (126) - : +- Sort (137) - : +- Exchange (136) - : +- Filter (135) - : +- Scan parquet (134) - +- Sort (145) - +- Exchange (144) - +- Filter (143) - +- Scan parquet (142) + Sort (142) + +- Exchange (141) + +- HashAggregate (140) + +- Exchange (139) + +- HashAggregate (138) + +- Project (137) + +- SortMergeJoin Inner (136) + :- Sort (131) + : +- Exchange (130) + : +- Project (129) + : +- SortMergeJoin Inner (128) + : :- Sort (123) + : : +- Exchange (122) + : : +- Project (121) + : : +- SortMergeJoin Inner (120) + : : :- Sort (115) + : : : +- Exchange (114) + : : : +- Project (113) + : : : +- SortMergeJoin Inner (112) + : : : :- Sort (107) + : : : : +- Exchange (106) + : : : : +- Project (105) + : : : : +- SortMergeJoin Inner (104) + : : : : :- Sort (99) + : : : : : +- Exchange (98) + : : : : : +- Project (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Sort (103) + : : : : +- Exchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- Sort (111) + : : : +- Exchange (110) + : : : +- Filter (109) + : : : +- Scan parquet (108) + : : +- Sort (119) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Sort (127) + : +- Exchange (126) + : +- Filter (125) + : +- Scan parquet (124) + +- Sort (135) + +- Exchange (134) + +- Filter (133) + +- Scan parquet (132) (1) Scan parquet @@ -164,614 +154,574 @@ Input [1]: [p_partkey#X] (8) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) SortExecTransformer -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(18) SortExecTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(24) InputAdapter -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(27) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(35) SortExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(41) InputAdapter -Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(42) InputIteratorTransformer +(37) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(43) SortExecTransformer +(38) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 -(44) Scan parquet +(39) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(50) InputAdapter +(45) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(51) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(52) SortExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 - -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(55) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(56) ColumnarExchange +(50) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(58) InputAdapter -Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] - -(59) InputIteratorTransformer +(52) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(60) SortExecTransformer +(53) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(61) Scan parquet +(54) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(62) NoopFilter +(55) NoopFilter Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_orderdate#X] -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(67) InputAdapter -Input [2]: [o_orderkey#X, o_orderdate#X] - -(68) InputIteratorTransformer +(60) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(69) SortExecTransformer +(61) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(75) InputAdapter -Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] - -(76) InputIteratorTransformer +(67) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(77) SortExecTransformer +(68) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(78) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) NoopFilter +(70) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(81) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(82) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(84) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(85) InputIteratorTransformer +(75) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(86) SortExecTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(87) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) ProjectExecTransformer +(78) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(89) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(90) ProjectExecTransformer +(80) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(91) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(82) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(83) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(84) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(85) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(98) ColumnarExchange +(88) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(99) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(100) InputAdapter +(90) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(101) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(102) SortExecTransformer +(92) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(103) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(104) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(105) Scan parquet +(95) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(106) Filter +(96) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(107) Project +(97) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(108) Exchange +(98) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Sort +(99) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(110) Scan parquet +(100) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(111) Filter +(101) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(112) Exchange +(102) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Sort +(103) Sort Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(114) SortMergeJoin +(104) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(115) Project +(105) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(116) Exchange +(106) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Sort +(107) Sort Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(118) Scan parquet +(108) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(119) Filter +(109) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(120) Exchange +(110) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) Sort +(111) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(122) SortMergeJoin +(112) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(123) Project +(113) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(124) Exchange +(114) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(115) Sort Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 -(126) Scan parquet +(116) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(127) Filter +(117) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(128) Exchange +(118) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(119) Sort Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 -(130) SortMergeJoin +(120) SortMergeJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(131) Project +(121) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(132) Exchange +(122) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(123) Sort Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(134) Scan parquet +(124) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(135) Filter +(125) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(136) Exchange +(126) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) Sort +(127) Sort Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(138) SortMergeJoin +(128) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(139) Project +(129) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(140) Exchange +(130) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) Sort +(131) Sort Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(142) Scan parquet +(132) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(143) Filter +(133) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(144) Exchange +(134) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Sort +(135) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(146) SortMergeJoin +(136) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(147) Project +(137) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(148) HashAggregate +(138) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(149) Exchange +(139) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(150) HashAggregate +(140) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(151) Exchange +(141) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Sort +(142) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(153) AdaptiveSparkPlan +(143) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt index 139349c6f401..9d1ce6d06e33 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt @@ -1,90 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- TakeOrderedAndProjectExecTransformer (65) - +- ^ ProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ FlushableHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - :- ^ SortExecTransformer (43) - : +- ^ InputIteratorTransformer (42) - : +- ShuffleQueryStage (40), Statistics(X) - : +- ColumnarExchange (39) - : +- ^ ProjectExecTransformer (37) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : :- ^ SortExecTransformer (26) - : : +- ^ InputIteratorTransformer (25) - : : +- ShuffleQueryStage (23), Statistics(X) - : : +- ColumnarExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : : :- ^ SortExecTransformer (9) - : : : +- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ SortExecTransformer (18) - : : +- ^ InputIteratorTransformer (17) - : : +- ShuffleQueryStage (15), Statistics(X) - : : +- ColumnarExchange (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ NoopFilter (11) - : : +- ^ Scan parquet (10) - : +- ^ SortExecTransformer (35) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ NoopFilter (28) - : +- ^ Scan parquet (27) - +- ^ SortExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ NoopFilter (45) - +- ^ Scan parquet (44) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + :- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : :- ^ InputIteratorTransformer (23) + : : +- ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- SortMergeJoin Inner (93) - :- Sort (88) - : +- Exchange (87) - : +- Project (86) - : +- SortMergeJoin Inner (85) - : :- Sort (79) - : : +- Exchange (78) - : : +- Project (77) - : : +- SortMergeJoin Inner (76) - : : :- Sort (70) - : : : +- Exchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- Sort (75) - : : +- Exchange (74) - : : +- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Sort (84) - : +- Exchange (83) - : +- Project (82) - : +- Filter (81) - : +- Scan parquet (80) - +- Sort (92) - +- Exchange (91) - +- Filter (90) - +- Scan parquet (89) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- SortMergeJoin Inner (87) + :- Sort (82) + : +- Exchange (81) + : +- Project (80) + : +- SortMergeJoin Inner (79) + : :- Sort (73) + : : +- Exchange (72) + : : +- Project (71) + : : +- SortMergeJoin Inner (70) + : : :- Sort (64) + : : : +- Exchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- Sort (69) + : : +- Exchange (68) + : : +- Project (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Sort (78) + : +- Exchange (77) + : +- Project (76) + : +- Filter (75) + : +- Scan parquet (74) + +- Sort (86) + +- Exchange (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -120,392 +114,368 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct (8) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) SortExecTransformer -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(18) SortExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(24) InputAdapter -Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(27) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(35) SortExecTransformer -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(41) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(42) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(43) SortExecTransformer -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 - -(44) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(50) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(51) InputIteratorTransformer +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(52) SortExecTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(55) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) ProjectExecTransformer +(50) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(57) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(58) ColumnarExchange +(52) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(53) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(60) InputAdapter +(54) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(61) InputIteratorTransformer +(55) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(62) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(63) ProjectExecTransformer +(57) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(64) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(65) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(66) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) Scan parquet +(61) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(62) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) Exchange +(63) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) Sort +(64) Sort Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(71) Scan parquet +(65) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(72) Filter +(66) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(73) Project +(67) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(74) Exchange +(68) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) Sort +(69) Sort Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(76) SortMergeJoin +(70) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(77) Project +(71) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(78) Exchange +(72) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Sort +(73) Sort Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(80) Scan parquet +(74) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(81) Filter +(75) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(82) Project +(76) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(83) Exchange +(77) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Sort +(78) Sort Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(85) SortMergeJoin +(79) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(86) Project +(80) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(87) Exchange +(81) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Sort +(82) Sort Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(89) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(90) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(91) Exchange +(85) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(86) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(93) SortMergeJoin +(87) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(94) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(95) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(96) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(98) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(99) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt index 6feb642e5976..efccadc766bb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt @@ -1,73 +1,69 @@ == Physical Plan == -AdaptiveSparkPlan (80) +AdaptiveSparkPlan (76) +- == Final Plan == - VeloxColumnarToRowExec (54) - +- ^ SortExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ FilterExecTransformer (46) - +- ^ RegularHashAggregateExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ShuffleQueryStage (42), Statistics(X) - +- ColumnarExchange (41) - +- ^ ProjectExecTransformer (39) - +- ^ FlushableHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - :- ^ SortExecTransformer (26) - : +- ^ InputIteratorTransformer (25) - : +- ShuffleQueryStage (23), Statistics(X) - : +- ColumnarExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ NoopFilter (11) - : +- ^ Scan parquet (10) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- ^ ProjectExecTransformer (29) - +- ^ NoopFilter (28) - +- ^ Scan parquet (27) + VeloxColumnarToRowExec (50) + +- ^ SortExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ShuffleQueryStage (38), Statistics(X) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + :- ^ InputIteratorTransformer (23) + : +- ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - Sort (79) - +- Exchange (78) - +- Filter (77) - +- HashAggregate (76) - +- Exchange (75) - +- HashAggregate (74) - +- Project (73) - +- SortMergeJoin Inner (72) - :- Sort (66) - : +- Exchange (65) - : +- Project (64) - : +- SortMergeJoin Inner (63) - : :- Sort (58) - : : +- Exchange (57) - : : +- Filter (56) - : : +- Scan parquet (55) - : +- Sort (62) - : +- Exchange (61) - : +- Filter (60) - : +- Scan parquet (59) - +- Sort (71) - +- Exchange (70) - +- Project (69) - +- Filter (68) - +- Scan parquet (67) + Sort (75) + +- Exchange (74) + +- Filter (73) + +- HashAggregate (72) + +- Exchange (71) + +- HashAggregate (70) + +- Project (69) + +- SortMergeJoin Inner (68) + :- Sort (62) + : +- Exchange (61) + : +- Project (60) + : +- SortMergeJoin Inner (59) + : :- Sort (54) + : : +- Exchange (53) + : : +- Filter (52) + : : +- Scan parquet (51) + : +- Sort (58) + : +- Exchange (57) + : +- Filter (56) + : +- Scan parquet (55) + +- Sort (67) + +- Exchange (66) + +- Project (65) + +- Filter (64) + +- Scan parquet (63) (1) Scan parquet @@ -103,607 +99,571 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] (8) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) SortExecTransformer -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(18) SortExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(26) SortExecTransformer -Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(27) Scan parquet +(24) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(33) InputAdapter -Input [1]: [n_nationkey#X] - -(34) InputIteratorTransformer +(30) InputAdapter Input [1]: [n_nationkey#X] -(35) SortExecTransformer +(31) InputIteratorTransformer Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(38) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(39) ProjectExecTransformer +(35) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(41) ColumnarExchange +(37) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(38) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(43) InputAdapter +(39) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(44) InputIteratorTransformer +(40) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(45) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(46) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(47) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(48) ColumnarExchange +(44) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(50) InputAdapter +(46) InputAdapter Input [2]: [ps_partkey#X, value#X] -(51) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(52) SortExecTransformer +(48) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(53) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(54) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(55) Scan parquet +(51) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(56) Filter +(52) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(57) Exchange +(53) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) Sort +(54) Sort Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 -(59) Scan parquet +(55) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(56) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) Exchange +(57) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) Sort +(58) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(63) SortMergeJoin +(59) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(64) Project +(60) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(65) Exchange +(61) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Sort +(62) Sort Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(67) Scan parquet +(63) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(68) Filter +(64) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(69) Project +(65) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(70) Exchange +(66) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(67) Sort Input [1]: [n_nationkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(72) SortMergeJoin +(68) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(73) Project +(69) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(74) HashAggregate +(70) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(75) Exchange +(71) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(77) Filter +(73) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(78) Exchange +(74) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Sort +(75) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(80) AdaptiveSparkPlan +(76) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (136) +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (114) - +- ^ ProjectExecTransformer (112) - +- ^ RegularHashAggregateExecTransformer (111) - +- ^ RegularHashAggregateExecTransformer (110) - +- ^ ProjectExecTransformer (109) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (108) - :- ^ SortExecTransformer (102) - : +- ^ InputIteratorTransformer (101) - : +- ShuffleQueryStage (99), Statistics(X) - : +- ColumnarExchange (98) - : +- ^ ProjectExecTransformer (96) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (95) - : :- ^ SortExecTransformer (89) - : : +- ^ InputIteratorTransformer (88) - : : +- ShuffleQueryStage (86), Statistics(X) - : : +- ColumnarExchange (85) - : : +- ^ ProjectExecTransformer (83) - : : +- ^ NoopFilter (82) - : : +- ^ Scan parquet (81) - : +- ^ SortExecTransformer (94) - : +- ^ InputIteratorTransformer (93) - : +- ShuffleQueryStage (91), Statistics(X) - : +- ReusedExchange (90) - +- ^ SortExecTransformer (107) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ReusedExchange (103) + VeloxColumnarToRowExec (106) + +- ^ ProjectExecTransformer (104) + +- ^ RegularHashAggregateExecTransformer (103) + +- ^ RegularHashAggregateExecTransformer (102) + +- ^ ProjectExecTransformer (101) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (100) + :- ^ InputIteratorTransformer (95) + : +- ShuffleQueryStage (93), Statistics(X) + : +- ColumnarExchange (92) + : +- ^ ProjectExecTransformer (90) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (89) + : :- ^ InputIteratorTransformer (84) + : : +- ShuffleQueryStage (82), Statistics(X) + : : +- ColumnarExchange (81) + : : +- ^ ProjectExecTransformer (79) + : : +- ^ NoopFilter (78) + : : +- ^ Scan parquet (77) + : +- ^ InputIteratorTransformer (88) + : +- ShuffleQueryStage (86), Statistics(X) + : +- ReusedExchange (85) + +- ^ InputIteratorTransformer (99) + +- ShuffleQueryStage (97), Statistics(X) + +- ReusedExchange (96) +- == Initial Plan == - HashAggregate (135) - +- HashAggregate (134) - +- Project (133) - +- SortMergeJoin Inner (132) - :- Sort (126) - : +- Exchange (125) - : +- Project (124) - : +- SortMergeJoin Inner (123) - : :- Sort (118) - : : +- Exchange (117) - : : +- Filter (116) - : : +- Scan parquet (115) - : +- Sort (122) - : +- Exchange (121) - : +- Filter (120) - : +- Scan parquet (119) - +- Sort (131) - +- Exchange (130) - +- Project (129) - +- Filter (128) - +- Scan parquet (127) - - -(81) Scan parquet + HashAggregate (127) + +- HashAggregate (126) + +- Project (125) + +- SortMergeJoin Inner (124) + :- Sort (118) + : +- Exchange (117) + : +- Project (116) + : +- SortMergeJoin Inner (115) + : :- Sort (110) + : : +- Exchange (109) + : : +- Filter (108) + : : +- Scan parquet (107) + : +- Sort (114) + : +- Exchange (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Sort (123) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) + + +(77) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(82) NoopFilter +(78) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(83) ProjectExecTransformer +(79) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(84) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(85) ColumnarExchange +(81) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(86) ShuffleQueryStage +(82) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(87) InputAdapter +(83) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(88) InputIteratorTransformer +(84) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(89) SortExecTransformer -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 - -(90) ReusedExchange [Reuses operator id: 14] +(85) ReusedExchange [Reuses operator id: 13] Output [2]: [s_suppkey#X, s_nationkey#X] -(91) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(92) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(93) InputIteratorTransformer +(87) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(94) SortExecTransformer +(88) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(95) ShuffledHashJoinExecTransformer +(89) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(96) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(97) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(98) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(99) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(100) InputAdapter -Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] - -(101) InputIteratorTransformer +(94) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(102) SortExecTransformer +(95) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(103) ReusedExchange [Reuses operator id: 31] +(96) ReusedExchange [Reuses operator id: 28] Output [1]: [n_nationkey#X] -(104) ShuffleQueryStage +(97) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(105) InputAdapter +(98) InputAdapter Input [1]: [n_nationkey#X] -(106) InputIteratorTransformer +(99) InputIteratorTransformer Input [1]: [n_nationkey#X] -(107) SortExecTransformer -Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(108) ShuffledHashJoinExecTransformer +(100) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(109) ProjectExecTransformer +(101) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(110) RegularHashAggregateExecTransformer +(102) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(111) RegularHashAggregateExecTransformer +(103) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(112) ProjectExecTransformer +(104) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(113) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(114) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(115) Scan parquet +(107) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(116) Filter +(108) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(117) Exchange +(109) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Sort +(110) Sort Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 -(119) Scan parquet +(111) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(120) Filter +(112) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(121) Exchange +(113) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(114) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(123) SortMergeJoin +(115) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(124) Project +(116) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(125) Exchange +(117) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) Sort +(118) Sort Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(127) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(128) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(129) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(130) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) Sort +(123) Sort Input [1]: [n_nationkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(132) SortMergeJoin +(124) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(133) Project +(125) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(134) HashAggregate +(126) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(135) HashAggregate +(127) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(136) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt index f6938aaf5ccf..d276d2ae31e1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt @@ -1,50 +1,48 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (51) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- SortMergeJoin Inner (46) - :- Sort (40) - : +- Exchange (39) - : +- Filter (38) - : +- Scan parquet (37) - +- Sort (45) - +- Exchange (44) - +- Project (43) - +- Filter (42) - +- Scan parquet (41) + Sort (50) + +- Exchange (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- Project (45) + +- SortMergeJoin Inner (44) + :- Sort (38) + : +- Exchange (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Sort (43) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -80,198 +78,190 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] (8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) SortExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(18) SortExecTransformer -Input [2]: [l_orderkey#X, l_shipmode#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(21) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(24) ColumnarExchange +(22) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(26) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(27) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(28) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(29) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(30) ColumnarExchange +(28) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(32) InputAdapter +(30) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(33) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(34) SortExecTransformer +(32) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(36) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(37) Scan parquet +(35) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(38) Filter +(36) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(39) Exchange +(37) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(38) Sort Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(41) Scan parquet +(39) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(42) Filter +(40) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(43) Project +(41) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(44) Exchange +(42) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(43) Sort Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(46) SortMergeJoin +(44) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(47) Project +(45) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(48) HashAggregate +(46) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(49) Exchange +(47) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(48) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(51) Exchange +(49) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(50) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(51) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt index 052c43c651b9..162ea40309f1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt @@ -1,53 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) - :- ^ SortExecTransformer (8) - : +- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) + :- ^ InputIteratorTransformer (7) + : +- ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- Project (48) - +- SortMergeJoin LeftOuter (47) - :- Sort (41) - : +- Exchange (40) - : +- Scan parquet (39) - +- Sort (46) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftOuter (45) + :- Sort (39) + : +- Exchange (38) + : +- Scan parquet (37) + +- Sort (44) + +- Exchange (43) + +- Project (42) + +- Filter (41) + +- Scan parquet (40) (1) Scan parquet @@ -78,225 +76,217 @@ Input [1]: [c_custkey#X] (7) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) SortExecTransformer -Input [1]: [c_custkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(9) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(10) NoopFilter +(9) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) ProjectExecTransformer +(10) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(12) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(12) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(13) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(14) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(15) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) SortExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(18) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(19) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(20) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(21) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(23) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(24) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(25) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(26) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(28) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(29) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(30) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(31) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(32) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(34) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(35) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(36) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(38) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(39) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(40) Exchange +(38) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(39) Sort Input [1]: [c_custkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(42) Scan parquet +(40) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(43) Filter +(41) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(44) Project +(42) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(45) Exchange +(43) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(44) Sort Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(47) SortMergeJoin +(45) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(49) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(50) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(51) HashAggregate +(49) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(52) Exchange +(50) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(51) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(54) Exchange +(52) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(53) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(56) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt index d3570649f11a..6ee3dab1c0dc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt @@ -1,40 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- ^ ProjectExecTransformer (23) - +- ^ RegularHashAggregateExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (38) - +- HashAggregate (37) - +- Project (36) - +- SortMergeJoin Inner (35) - :- Sort (30) - : +- Exchange (29) - : +- Project (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Sort (34) - +- Exchange (33) - +- Filter (32) - +- Scan parquet (31) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- SortMergeJoin Inner (33) + :- Sort (28) + : +- Exchange (27) + : +- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- Sort (32) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -70,146 +68,138 @@ Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] (8) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) SortExecTransformer -Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X, p_type#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(18) SortExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(21) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(22) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(23) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(24) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(25) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(26) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(27) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(28) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(29) Exchange +(27) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) Sort +(28) Sort Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(31) Scan parquet +(29) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(32) Filter +(30) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(33) Exchange +(31) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Sort +(32) Sort Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(35) SortMergeJoin +(33) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(36) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(37) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(38) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(39) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt index ea533e3c0bd6..34027baf1292 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt @@ -1,47 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- AQEShuffleRead (28) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (22) - +- ^ FilterExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- Project (43) - +- SortMergeJoin Inner (42) - :- Sort (33) - : +- Exchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Sort (41) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (43) + +- Exchange (42) + +- Project (41) + +- SortMergeJoin Inner (40) + :- Sort (31) + : +- Exchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Sort (39) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -77,330 +75,322 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] (8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) SortExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(13) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) ProjectExecTransformer +(13) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(15) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(16) ColumnarExchange +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(18) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(21) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(22) SortExecTransformer -Input [2]: [supplier_no#X, total_revenue#X] -Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 - -(23) ShuffledHashJoinExecTransformer +(21) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(24) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(25) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(26) ColumnarExchange +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(28) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(29) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) Exchange +(30) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) Sort +(31) Sort Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(34) Scan parquet +(32) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(33) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(34) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(36) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(40) Filter +(38) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) Sort +(39) Sort Input [2]: [supplier_no#X, total_revenue#X] Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 -(42) SortMergeJoin +(40) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(43) Project +(41) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(44) Exchange +(42) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(43) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (71) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (62) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ ProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ NoopFilter (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (48) + +- ^ ProjectExecTransformer (47) + +- ^ NoopFilter (46) + +- ^ Scan parquet (45) +- == Initial Plan == - HashAggregate (70) - +- HashAggregate (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- Filter (64) - +- Scan parquet (63) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(47) Scan parquet +(45) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(48) NoopFilter +(46) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(49) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(50) FlushableHashAggregateExecTransformer +(48) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(58) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(59) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(60) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(61) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(62) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(63) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(64) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(65) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(66) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(67) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(69) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(70) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(71) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt index f9a9e8b17709..acb3799c1df4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt @@ -1,63 +1,61 @@ == Physical Plan == -AdaptiveSparkPlan (68) +AdaptiveSparkPlan (66) +- == Final Plan == - VeloxColumnarToRowExec (44) - +- ^ SortExecTransformer (42) - +- ^ InputIteratorTransformer (41) - +- ShuffleQueryStage (39), Statistics(X) - +- ColumnarExchange (38) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ ProjectExecTransformer (30) - +- ^ FlushableHashAggregateExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (42) + +- ^ SortExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (67) - +- Exchange (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- HashAggregate (62) - +- Exchange (61) - +- HashAggregate (60) - +- Project (59) - +- SortMergeJoin Inner (58) - :- Sort (53) - : +- Exchange (52) - : +- BroadcastHashJoin LeftAnti BuildRight (51) - : :- Filter (46) - : : +- Scan parquet (45) - : +- BroadcastExchange (50) - : +- Project (49) - : +- Filter (48) - : +- Scan parquet (47) - +- Sort (57) - +- Exchange (56) - +- Filter (55) - +- Scan parquet (54) + Sort (65) + +- Exchange (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Exchange (59) + +- HashAggregate (58) + +- Project (57) + +- SortMergeJoin Inner (56) + :- Sort (51) + : +- Exchange (50) + : +- BroadcastHashJoin LeftAnti BuildRight (49) + : :- Filter (44) + : : +- Scan parquet (43) + : +- BroadcastExchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Sort (55) + +- Exchange (54) + +- Filter (53) + +- Scan parquet (52) (1) Scan parquet @@ -93,272 +91,264 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] (8) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) SortExecTransformer -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(18) SortExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(21) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(24) ColumnarExchange +(22) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(23) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(26) InputAdapter +(24) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) InputIteratorTransformer +(25) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(28) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(29) FlushableHashAggregateExecTransformer +(27) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(30) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(32) ColumnarExchange +(30) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(34) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(35) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(36) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(37) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(38) ColumnarExchange +(36) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(37) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(40) InputAdapter +(38) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(41) InputIteratorTransformer +(39) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) SortExecTransformer +(40) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(43) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(44) VeloxColumnarToRowExec +(42) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(45) Scan parquet +(43) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(46) Filter +(44) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(47) Scan parquet +(45) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(48) Filter +(46) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(49) Project +(47) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(50) BroadcastExchange +(48) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(51) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(52) Exchange +(50) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) Sort +(51) Sort Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 -(54) Scan parquet +(52) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(55) Filter +(53) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(56) Exchange +(54) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(55) Sort Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(58) SortMergeJoin +(56) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(59) Project +(57) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(60) HashAggregate +(58) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(61) Exchange +(59) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(60) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(63) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(64) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(66) Exchange +(64) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(67) Sort +(65) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(68) AdaptiveSparkPlan +(66) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt index 2d37c6bfed92..a20017e701d5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt @@ -1,62 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ ProjectExecTransformer (38) - +- ^ RegularHashAggregateExecTransformer (37) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ ProjectExecTransformer (35) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) - :- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ NoopFilter (11) - : +- ^ Scan parquet (10) - +- ^ SortExecTransformer (33) - +- ^ FilterExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ NoopFilter (22) - +- ^ Scan parquet (21) + VeloxColumnarToRowExec (37) + +- ^ ProjectExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ RegularHashAggregateExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) + :- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ FilterExecTransformer (30) + +- ^ ProjectExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ NoopFilter (20) + +- ^ Scan parquet (19) +- == Initial Plan == - HashAggregate (62) - +- HashAggregate (61) - +- Project (60) - +- SortMergeJoin Inner (59) - :- Project (51) - : +- SortMergeJoin Inner (50) - : :- Sort (44) - : : +- Exchange (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Sort (49) - : +- Exchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Sort (58) - +- Filter (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Filter (53) - +- Scan parquet (52) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- SortMergeJoin Inner (56) + :- Project (48) + : +- SortMergeJoin Inner (47) + : :- Sort (41) + : : +- Exchange (40) + : : +- Filter (39) + : : +- Scan parquet (38) + : +- Sort (46) + : +- Exchange (45) + : +- Project (44) + : +- Filter (43) + : +- Scan parquet (42) + +- Sort (55) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -92,260 +89,248 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] (8) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) SortExecTransformer -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_container#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [1]: [p_partkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [p_partkey#X] -(18) SortExecTransformer -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(21) Scan parquet +(19) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(22) NoopFilter +(20) NoopFilter Input [2]: [l_partkey#X, l_quantity#X] Arguments: [l_partkey#X, l_quantity#X] -(23) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(24) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(25) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(26) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(28) InputAdapter +(26) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(29) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(30) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(31) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(32) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(33) SortExecTransformer -Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(34) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(35) ProjectExecTransformer +(32) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(36) RegularHashAggregateExecTransformer +(33) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(37) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(38) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(39) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(40) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(41) Scan parquet +(38) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(42) Filter +(39) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(43) Exchange +(40) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(41) Sort Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(45) Scan parquet +(42) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(46) Filter +(43) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(47) Project +(44) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(48) Exchange +(45) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(46) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(50) SortMergeJoin +(47) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(51) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(52) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(53) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(54) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(55) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(57) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(58) Sort +(55) Sort Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(59) SortMergeJoin +(56) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(60) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(61) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(62) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(63) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt index a6f530e94a67..65c2bdf4162d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt @@ -1,103 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (111) +AdaptiveSparkPlan (104) +- == Final Plan == - VeloxColumnarToRowExec (71) - +- TakeOrderedAndProjectExecTransformer (70) - +- ^ RegularHashAggregateExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (67) - +- ^ ProjectExecTransformer (66) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (65) - :- ^ SortExecTransformer (46) - : +- ^ InputIteratorTransformer (45) - : +- ShuffleQueryStage (43), Statistics(X) - : +- ColumnarExchange (42) - : +- ^ ProjectExecTransformer (40) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (38) - : +- ^ InputIteratorTransformer (37) - : +- ShuffleQueryStage (35), Statistics(X) - : +- ColumnarExchange (34) - : +- ^ ProjectExecTransformer (32) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) - : :- ^ SortExecTransformer (18) - : : +- ^ InputIteratorTransformer (17) - : : +- ShuffleQueryStage (15), Statistics(X) - : : +- ColumnarExchange (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ NoopFilter (11) - : : +- ^ Scan parquet (10) - : +- ^ SortExecTransformer (30) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ RegularHashAggregateExecTransformer (27) - : +- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24), Statistics(X) - : +- ColumnarExchange (23) - : +- ^ ProjectExecTransformer (21) - : +- ^ FlushableHashAggregateExecTransformer (20) - : +- ^ Scan parquet (19) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (64) - :- ^ SortExecTransformer (55) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ SortExecTransformer (63) - +- ^ ProjectExecTransformer (62) - +- ^ FilterExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ InputIteratorTransformer (59) - +- ShuffleQueryStage (57), Statistics(X) - +- ReusedExchange (56) + VeloxColumnarToRowExec (64) + +- TakeOrderedAndProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) + :- ^ InputIteratorTransformer (41) + : +- ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : :- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) + :- ^ InputIteratorTransformer (49) + : +- ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (110) - +- HashAggregate (109) - +- HashAggregate (108) - +- Project (107) - +- SortMergeJoin Inner (106) - :- Sort (93) - : +- Exchange (92) - : +- Project (91) - : +- SortMergeJoin Inner (90) - : :- Sort (75) - : : +- Exchange (74) - : : +- Filter (73) - : : +- Scan parquet (72) - : +- Sort (89) - : +- Exchange (88) - : +- SortMergeJoin LeftSemi (87) - : :- Sort (79) - : : +- Exchange (78) - : : +- Filter (77) - : : +- Scan parquet (76) - : +- Sort (86) - : +- Project (85) - : +- Filter (84) - : +- HashAggregate (83) - : +- Exchange (82) - : +- HashAggregate (81) - : +- Scan parquet (80) - +- SortMergeJoin LeftSemi (105) - :- Sort (97) - : +- Exchange (96) - : +- Filter (95) - : +- Scan parquet (94) - +- Sort (104) - +- Project (103) - +- Filter (102) - +- HashAggregate (101) - +- Exchange (100) - +- HashAggregate (99) - +- Scan parquet (98) + TakeOrderedAndProject (103) + +- HashAggregate (102) + +- HashAggregate (101) + +- Project (100) + +- SortMergeJoin Inner (99) + :- Sort (86) + : +- Exchange (85) + : +- Project (84) + : +- SortMergeJoin Inner (83) + : :- Sort (68) + : : +- Exchange (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Sort (82) + : +- Exchange (81) + : +- SortMergeJoin LeftSemi (80) + : :- Sort (72) + : : +- Exchange (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- Sort (79) + : +- Project (78) + : +- Filter (77) + : +- HashAggregate (76) + : +- Exchange (75) + : +- HashAggregate (74) + : +- Scan parquet (73) + +- SortMergeJoin LeftSemi (98) + :- Sort (90) + : +- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Sort (97) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -133,462 +126,434 @@ Input [2]: [c_custkey#X, c_name#X] (8) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) SortExecTransformer -Input [2]: [c_custkey#X, c_name#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(16) InputAdapter -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(17) InputIteratorTransformer +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(18) SortExecTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(19) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(20) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(28) FilterExecTransformer +(26) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(29) ProjectExecTransformer +(27) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(30) SortExecTransformer -Input [1]: [l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(31) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(32) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(33) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(34) ColumnarExchange +(31) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(36) InputAdapter +(33) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(38) SortExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(39) ShuffledHashJoinExecTransformer +(35) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) ProjectExecTransformer +(36) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(41) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(42) ColumnarExchange +(38) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(39) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(44) InputAdapter +(40) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(45) InputIteratorTransformer +(41) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(46) SortExecTransformer -Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(47) Scan parquet +(42) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(48) NoopFilter +(43) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(49) ProjectExecTransformer +(44) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(50) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(51) ColumnarExchange +(46) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(47) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(53) InputAdapter +(48) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(54) InputIteratorTransformer +(49) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(55) SortExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(56) ReusedExchange [Reuses operator id: 23] +(50) ReusedExchange [Reuses operator id: 21] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(58) InputAdapter +(52) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(59) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(60) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(61) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(62) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(63) SortExecTransformer -Input [1]: [l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(64) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(65) ShuffledHashJoinExecTransformer +(58) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(66) ProjectExecTransformer +(59) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(67) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(68) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(69) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(70) TakeOrderedAndProjectExecTransformer +(63) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(71) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(72) Scan parquet +(65) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(73) Filter +(66) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(74) Exchange +(67) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) Sort +(68) Sort Input [2]: [c_custkey#X, c_name#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(76) Scan parquet +(69) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(77) Filter +(70) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(78) Exchange +(71) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Sort +(72) Sort Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(80) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(81) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(82) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(83) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(84) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(85) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(86) Sort +(79) Sort Input [1]: [l_orderkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(87) SortMergeJoin +(80) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(88) Exchange +(81) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Sort +(82) Sort Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(90) SortMergeJoin +(83) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(91) Project +(84) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(92) Exchange +(85) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(86) Sort Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(94) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(96) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(90) Sort Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(98) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(99) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(100) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(102) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(103) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(104) Sort +(97) Sort Input [1]: [l_orderkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(105) SortMergeJoin +(98) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(106) SortMergeJoin +(99) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(107) Project +(100) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(108) HashAggregate +(101) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(109) HashAggregate +(102) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(110) TakeOrderedAndProject +(103) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(111) AdaptiveSparkPlan +(104) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt index ba5097790377..bee057ef76c2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt @@ -1,39 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (24) - +- ^ RegularHashAggregateExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (37) - +- HashAggregate (36) - +- Project (35) - +- SortMergeJoin Inner (34) - :- Sort (29) - : +- Exchange (28) - : +- Project (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Sort (33) - +- Exchange (32) - +- Filter (31) - +- Scan parquet (30) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- SortMergeJoin Inner (32) + :- Sort (27) + : +- Exchange (26) + : +- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- Sort (31) + +- Exchange (30) + +- Filter (29) + +- Scan parquet (28) (1) Scan parquet @@ -69,142 +67,134 @@ Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] (8) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) SortExecTransformer -Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(18) SortExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(21) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(22) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(24) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(25) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(26) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(27) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(28) Exchange +(26) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Sort +(27) Sort Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(30) Scan parquet +(28) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(31) Filter +(29) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(32) Exchange +(30) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) Sort +(31) Sort Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(34) SortMergeJoin +(32) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(35) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(36) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(37) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(38) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt index 2c1b00750978..6f9c1f5303cf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt @@ -1,129 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (143) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (93) - +- AQEShuffleRead (92) - +- ShuffleQueryStage (91), Statistics(X) - +- ColumnarExchange (90) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ SortExecTransformer (77) - : +- ^ InputIteratorTransformer (76) - : +- ShuffleQueryStage (74), Statistics(X) - : +- ColumnarExchange (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (70) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ SortExecTransformer (35) - : : +- ^ InputIteratorTransformer (34) - : : +- ShuffleQueryStage (32), Statistics(X) - : : +- ColumnarExchange (31) - : : +- ^ ProjectExecTransformer (29) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : : :- ^ SortExecTransformer (18) - : : : +- ^ InputIteratorTransformer (17) - : : : +- ShuffleQueryStage (15), Statistics(X) - : : : +- ColumnarExchange (14) - : : : +- ^ ProjectExecTransformer (12) - : : : +- ^ NoopFilter (11) - : : : +- ^ Scan parquet (10) - : : +- ^ SortExecTransformer (27) - : : +- ^ InputIteratorTransformer (26) - : : +- ShuffleQueryStage (24), Statistics(X) - : : +- ColumnarExchange (23) - : : +- ^ ProjectExecTransformer (21) - : : +- ^ NoopFilter (20) - : : +- ^ Scan parquet (19) - : +- ^ SortExecTransformer (61) - : +- ^ InputIteratorTransformer (60) - : +- ShuffleQueryStage (58), Statistics(X) - : +- ColumnarExchange (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ FilterExecTransformer (54) - : +- ^ ProjectExecTransformer (53) - : +- ^ RegularHashAggregateExecTransformer (52) - : +- ^ RegularHashAggregateExecTransformer (51) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (50) - : :- ^ SortExecTransformer (44) - : : +- ^ InputIteratorTransformer (43) - : : +- ShuffleQueryStage (41), Statistics(X) - : : +- ColumnarExchange (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ NoopFilter (37) - : : +- ^ Scan parquet (36) - : +- ^ SortExecTransformer (49) - : +- ^ InputIteratorTransformer (48) - : +- ShuffleQueryStage (46), Statistics(X) - : +- ReusedExchange (45) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (83) + +- AQEShuffleRead (82) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) + : :- ^ InputIteratorTransformer (31) + : : +- ShuffleQueryStage (29), Statistics(X) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ShuffleQueryStage (14), Statistics(X) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ NoopFilter (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ShuffleQueryStage (22), Statistics(X) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ NoopFilter (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ NoopFilter (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ReusedExchange (40) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (142) - +- Exchange (141) - +- Project (140) - +- SortMergeJoin Inner (139) - :- Sort (133) - : +- Exchange (132) - : +- Project (131) - : +- SortMergeJoin LeftSemi (130) - : :- Sort (97) - : : +- Exchange (96) - : : +- Filter (95) - : : +- Scan parquet (94) - : +- Sort (129) - : +- Exchange (128) - : +- Project (127) - : +- SortMergeJoin Inner (126) - : :- Sort (109) - : : +- Exchange (108) - : : +- SortMergeJoin LeftSemi (107) - : : :- Sort (101) - : : : +- Exchange (100) - : : : +- Filter (99) - : : : +- Scan parquet (98) - : : +- Sort (106) - : : +- Exchange (105) - : : +- Project (104) - : : +- Filter (103) - : : +- Scan parquet (102) - : +- Sort (125) - : +- Exchange (124) - : +- Filter (123) - : +- HashAggregate (122) - : +- HashAggregate (121) - : +- SortMergeJoin LeftSemi (120) - : :- Sort (114) - : : +- Exchange (113) - : : +- Project (112) - : : +- Filter (111) - : : +- Scan parquet (110) - : +- Sort (119) - : +- Exchange (118) - : +- Project (117) - : +- Filter (116) - : +- Scan parquet (115) - +- Sort (138) - +- Exchange (137) - +- Project (136) - +- Filter (135) - +- Scan parquet (134) + Sort (132) + +- Exchange (131) + +- Project (130) + +- SortMergeJoin Inner (129) + :- Sort (123) + : +- Exchange (122) + : +- Project (121) + : +- SortMergeJoin LeftSemi (120) + : :- Sort (87) + : : +- Exchange (86) + : : +- Filter (85) + : : +- Scan parquet (84) + : +- Sort (119) + : +- Exchange (118) + : +- Project (117) + : +- SortMergeJoin Inner (116) + : :- Sort (99) + : : +- Exchange (98) + : : +- SortMergeJoin LeftSemi (97) + : : :- Sort (91) + : : : +- Exchange (90) + : : : +- Filter (89) + : : : +- Scan parquet (88) + : : +- Sort (96) + : : +- Exchange (95) + : : +- Project (94) + : : +- Filter (93) + : : +- Scan parquet (92) + : +- Sort (115) + : +- Exchange (114) + : +- Filter (113) + : +- HashAggregate (112) + : +- HashAggregate (111) + : +- SortMergeJoin LeftSemi (110) + : :- Sort (104) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Sort (109) + : +- Exchange (108) + : +- Project (107) + : +- Filter (106) + : +- Scan parquet (105) + +- Sort (128) + +- Exchange (127) + +- Project (126) + +- Filter (125) + +- Scan parquet (124) (1) Scan parquet @@ -159,574 +149,534 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] (8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) SortExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(16) InputAdapter -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(17) InputIteratorTransformer +(15) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(18) SortExecTransformer +(16) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 -(19) Scan parquet +(17) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(20) NoopFilter +(18) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [1]: [p_partkey#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [1]: [p_partkey#X] -(27) SortExecTransformer -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(28) ShuffledHashJoinExecTransformer +(25) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(35) SortExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 - -(36) Scan parquet +(32) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(37) NoopFilter +(33) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(38) ProjectExecTransformer +(34) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(39) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(40) ColumnarExchange +(36) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(41) ShuffleQueryStage +(37) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(42) InputAdapter +(38) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(43) InputIteratorTransformer +(39) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(44) SortExecTransformer -Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(45) ReusedExchange [Reuses operator id: 23] +(40) ReusedExchange [Reuses operator id: 21] Output [1]: [p_partkey#X] -(46) ShuffleQueryStage +(41) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(47) InputAdapter -Input [1]: [p_partkey#X] - -(48) InputIteratorTransformer +(42) InputAdapter Input [1]: [p_partkey#X] -(49) SortExecTransformer +(43) InputIteratorTransformer Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(50) ShuffledHashJoinExecTransformer +(44) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(51) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(53) ProjectExecTransformer +(47) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(54) FilterExecTransformer +(48) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(55) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(56) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(57) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(59) InputAdapter +(53) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(60) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(61) SortExecTransformer -Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 - -(62) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(67) InputAdapter +(60) InputAdapter Input [1]: [ps_suppkey#X] -(68) InputIteratorTransformer -Input [1]: [ps_suppkey#X] - -(69) SortExecTransformer +(61) InputIteratorTransformer Input [1]: [ps_suppkey#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(75) InputAdapter +(67) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(76) InputIteratorTransformer +(68) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(77) SortExecTransformer -Input [3]: [s_name#X, s_address#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(78) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(79) NoopFilter +(70) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(81) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(82) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(84) InputAdapter +(75) InputAdapter Input [1]: [n_nationkey#X] -(85) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [n_nationkey#X] -(86) SortExecTransformer -Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(87) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) ProjectExecTransformer +(78) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(89) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(90) ColumnarExchange +(80) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(91) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(92) AQEShuffleRead +(82) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(93) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(94) Scan parquet +(84) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(95) Filter +(85) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(96) Exchange +(86) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(87) Sort Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(98) Scan parquet +(88) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(99) Filter +(89) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(100) Exchange +(90) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Sort +(91) Sort Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 -(102) Scan parquet +(92) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(103) Filter +(93) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(104) Project +(94) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(105) Exchange +(95) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Sort +(96) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(107) SortMergeJoin +(97) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(108) Exchange +(98) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Sort +(99) Sort Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 -(110) Scan parquet +(100) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(111) Filter +(101) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(112) Project +(102) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(113) Exchange +(103) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) Sort +(104) Sort Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(115) Scan parquet +(105) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(116) Filter +(106) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(117) Project +(107) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(118) Exchange +(108) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) Sort +(109) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(120) SortMergeJoin +(110) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(121) HashAggregate +(111) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(122) HashAggregate +(112) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(123) Filter +(113) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(124) Exchange +(114) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(115) Sort Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 -(126) SortMergeJoin +(116) SortMergeJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(127) Project +(117) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(128) Exchange +(118) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(119) Sort Input [1]: [ps_suppkey#X] Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 -(130) SortMergeJoin +(120) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(131) Project +(121) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(132) Exchange +(122) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(123) Sort Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(134) Scan parquet +(124) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(135) Filter +(125) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(136) Project +(126) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(137) Exchange +(127) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) Sort +(128) Sort Input [1]: [n_nationkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(139) SortMergeJoin +(129) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(140) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(141) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(143) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt index 8616f14166c3..f75abb503b69 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt @@ -1,121 +1,112 @@ == Physical Plan == -AdaptiveSparkPlan (136) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ RegularHashAggregateExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ ProjectExecTransformer (82) - +- ^ FlushableHashAggregateExecTransformer (81) - +- ^ ProjectExecTransformer (80) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) - :- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ SortExecTransformer (52) - : : +- ^ InputIteratorTransformer (51) - : : +- ShuffleQueryStage (49), Statistics(X) - : : +- ColumnarExchange (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) - : : :- ^ SortExecTransformer (9) - : : : +- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ SortExecTransformer (44) - : : +- ^ InputIteratorTransformer (43) - : : +- ShuffleQueryStage (41), Statistics(X) - : : +- ColumnarExchange (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) - : : : :- ^ SortExecTransformer (18) - : : : : +- ^ InputIteratorTransformer (17) - : : : : +- ShuffleQueryStage (15), Statistics(X) - : : : : +- ColumnarExchange (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ SortExecTransformer (26) - : : : +- ^ InputIteratorTransformer (25) - : : : +- ShuffleQueryStage (23), Statistics(X) - : : : +- ColumnarExchange (22) - : : : +- ^ ProjectExecTransformer (20) - : : : +- ^ Scan parquet (19) - : : +- ^ SortExecTransformer (36) - : : +- ^ InputIteratorTransformer (35) - : : +- ShuffleQueryStage (33), Statistics(X) - : : +- ColumnarExchange (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ NoopFilter (29) - : : +- ^ Scan parquet (28) - : +- ^ SortExecTransformer (61) - : +- ^ InputIteratorTransformer (60) - : +- ShuffleQueryStage (58), Statistics(X) - : +- ColumnarExchange (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ NoopFilter (54) - : +- ^ Scan parquet (53) - +- ^ SortExecTransformer (78) - +- ^ InputIteratorTransformer (77) - +- ShuffleQueryStage (75), Statistics(X) - +- ColumnarExchange (74) - +- ^ ProjectExecTransformer (72) - +- ^ NoopFilter (71) - +- ^ Scan parquet (70) + VeloxColumnarToRowExec (81) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + :- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) + : :- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ShuffleQueryStage (21), Statistics(X) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ShuffleQueryStage (30), Statistics(X) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == - TakeOrderedAndProject (135) - +- HashAggregate (134) - +- Exchange (133) - +- HashAggregate (132) - +- Project (131) - +- SortMergeJoin Inner (130) - :- Sort (124) - : +- Exchange (123) - : +- Project (122) - : +- SortMergeJoin Inner (121) - : :- Sort (115) - : : +- Exchange (114) - : : +- Project (113) - : : +- SortMergeJoin Inner (112) - : : :- Sort (94) - : : : +- Exchange (93) - : : : +- Filter (92) - : : : +- Scan parquet (91) - : : +- Sort (111) - : : +- Exchange (110) - : : +- SortMergeJoin LeftAnti (109) - : : :- SortMergeJoin LeftSemi (103) - : : : :- Sort (99) - : : : : +- Exchange (98) - : : : : +- Project (97) - : : : : +- Filter (96) - : : : : +- Scan parquet (95) - : : : +- Sort (102) - : : : +- Exchange (101) - : : : +- Scan parquet (100) - : : +- Sort (108) - : : +- Exchange (107) - : : +- Project (106) - : : +- Filter (105) - : : +- Scan parquet (104) - : +- Sort (120) - : +- Exchange (119) - : +- Project (118) - : +- Filter (117) - : +- Scan parquet (116) - +- Sort (129) - +- Exchange (128) - +- Project (127) - +- Filter (126) - +- Scan parquet (125) + TakeOrderedAndProject (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- SortMergeJoin Inner (121) + :- Sort (115) + : +- Exchange (114) + : +- Project (113) + : +- SortMergeJoin Inner (112) + : :- Sort (106) + : : +- Exchange (105) + : : +- Project (104) + : : +- SortMergeJoin Inner (103) + : : :- Sort (85) + : : : +- Exchange (84) + : : : +- Filter (83) + : : : +- Scan parquet (82) + : : +- Sort (102) + : : +- Exchange (101) + : : +- SortMergeJoin LeftAnti (100) + : : :- SortMergeJoin LeftSemi (94) + : : : :- Sort (90) + : : : : +- Exchange (89) + : : : : +- Project (88) + : : : : +- Filter (87) + : : : : +- Scan parquet (86) + : : : +- Sort (93) + : : : +- Exchange (92) + : : : +- Scan parquet (91) + : : +- Sort (99) + : : +- Exchange (98) + : : +- Project (97) + : : +- Filter (96) + : : +- Scan parquet (95) + : +- Sort (111) + : +- Exchange (110) + : +- Project (109) + : +- Filter (108) + : +- Scan parquet (107) + +- Sort (120) + +- Exchange (119) + +- Project (118) + +- Filter (117) + +- Scan parquet (116) (1) Scan parquet @@ -151,548 +142,512 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] (8) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) SortExecTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(18) SortExecTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(19) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(24) InputAdapter -Input [2]: [l_orderkey#X, l_suppkey#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(27) ShuffledHashJoinExecTransformer +(24) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(28) Scan parquet +(25) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(29) NoopFilter +(26) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(30) ProjectExecTransformer +(27) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(31) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(32) ColumnarExchange +(29) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(34) InputAdapter +(31) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(35) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(36) SortExecTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(37) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(38) ProjectExecTransformer +(34) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(40) ColumnarExchange +(36) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(41) ShuffleQueryStage +(37) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(42) InputAdapter +(38) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(43) InputIteratorTransformer +(39) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(44) SortExecTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 - -(45) ShuffledHashJoinExecTransformer +(40) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(50) InputAdapter -Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] - -(51) InputIteratorTransformer +(45) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(52) SortExecTransformer +(46) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(53) Scan parquet +(47) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(54) NoopFilter +(48) NoopFilter Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: [o_orderkey#X, o_orderstatus#X] -(55) ProjectExecTransformer +(49) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(56) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(57) ColumnarExchange +(51) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(52) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(59) InputAdapter +(53) InputAdapter Input [1]: [o_orderkey#X] -(60) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [o_orderkey#X] -(61) SortExecTransformer -Input [1]: [o_orderkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(62) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(60) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(68) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(69) SortExecTransformer -Input [2]: [s_name#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(70) Scan parquet +(62) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(71) NoopFilter +(63) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(72) ProjectExecTransformer +(64) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(73) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(74) ColumnarExchange +(66) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(75) ShuffleQueryStage +(67) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(76) InputAdapter -Input [1]: [n_nationkey#X] - -(77) InputIteratorTransformer +(68) InputAdapter Input [1]: [n_nationkey#X] -(78) SortExecTransformer +(69) InputIteratorTransformer Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(79) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(81) FlushableHashAggregateExecTransformer +(72) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(82) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(83) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(84) ColumnarExchange +(75) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(86) InputAdapter +(77) InputAdapter Input [2]: [s_name#X, count#X] -(87) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(88) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(89) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(90) VeloxColumnarToRowExec +(81) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(91) Scan parquet +(82) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(83) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(84) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Sort +(85) Sort Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(95) Scan parquet +(86) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(96) Filter +(87) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(97) Project +(88) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(98) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(90) Sort Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(100) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(101) Exchange +(92) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Sort +(93) Sort Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(103) SortMergeJoin +(94) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(104) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(105) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(106) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(107) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Sort +(99) Sort Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(109) SortMergeJoin +(100) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(110) Exchange +(101) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Sort +(102) Sort Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(112) SortMergeJoin +(103) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(113) Project +(104) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(114) Exchange +(105) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Sort +(106) Sort Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(116) Scan parquet +(107) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(117) Filter +(108) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(118) Project +(109) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(119) Exchange +(110) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(111) Sort Input [1]: [o_orderkey#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(121) SortMergeJoin +(112) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(122) Project +(113) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(123) Exchange +(114) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) Sort +(115) Sort Input [2]: [s_name#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(125) Scan parquet +(116) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(126) Filter +(117) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(127) Project +(118) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(128) Exchange +(119) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(120) Sort Input [1]: [n_nationkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(130) SortMergeJoin +(121) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(131) Project +(122) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(132) HashAggregate +(123) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(133) Exchange +(124) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) HashAggregate +(125) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(135) TakeOrderedAndProject +(126) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(136) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt index 1f40861adcaa..110a3336271b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt @@ -1,47 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (48) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- SortMergeJoin LeftAnti (43) - :- Sort (39) - : +- Exchange (38) - : +- Filter (37) - : +- Scan parquet (36) - +- Sort (42) - +- Exchange (41) - +- Scan parquet (40) + Sort (47) + +- Exchange (46) + +- HashAggregate (45) + +- Exchange (44) + +- HashAggregate (43) + +- Project (42) + +- SortMergeJoin LeftAnti (41) + :- Sort (37) + : +- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Sort (40) + +- Exchange (39) + +- Scan parquet (38) (1) Scan parquet @@ -77,298 +75,290 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] (8) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) SortExecTransformer -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(12) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(12) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(13) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(15) InputAdapter +(14) InputAdapter Input [1]: [o_custkey#X] -(16) InputIteratorTransformer +(15) InputIteratorTransformer Input [1]: [o_custkey#X] -(17) SortExecTransformer -Input [1]: [o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(18) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(19) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(20) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(28) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(29) ColumnarExchange +(27) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(31) InputAdapter +(29) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(32) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) SortExecTransformer +(31) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(35) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(36) Scan parquet +(34) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(37) Filter +(35) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(38) Exchange +(36) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(37) Sort Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(40) Scan parquet +(38) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(39) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Sort +(40) Sort Input [1]: [o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(43) SortMergeJoin +(41) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(42) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(45) HashAggregate +(43) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(46) Exchange +(44) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(45) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(48) Exchange +(46) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(47) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(48) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (62) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ InputIteratorTransformer (59) - +- ShuffleQueryStage (57), Statistics(X) - +- ColumnarExchange (56) - +- ^ FlushableHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ NoopFilter (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- ^ FlushableHashAggregateExecTransformer (52) + +- ^ ProjectExecTransformer (51) + +- ^ NoopFilter (50) + +- ^ Scan parquet (49) +- == Initial Plan == - HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- Filter (64) - +- Scan parquet (63) + HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(51) Scan parquet +(49) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) NoopFilter +(50) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(53) ProjectExecTransformer +(51) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) FlushableHashAggregateExecTransformer +(52) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(56) ColumnarExchange +(54) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(55) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(58) InputAdapter +(56) InputAdapter Input [2]: [sum#X, count#X] -(59) InputIteratorTransformer +(57) InputIteratorTransformer Input [2]: [sum#X, count#X] -(60) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(61) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(62) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(63) Scan parquet +(61) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(64) Filter +(62) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(65) Project +(63) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(66) HashAggregate +(64) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(67) Exchange +(65) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(66) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(69) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt index cfaf52ba01d5..a416e3379d62 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt @@ -1,64 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ RegularHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - :- ^ SortExecTransformer (26) - : +- ^ InputIteratorTransformer (25) - : +- ShuffleQueryStage (23), Statistics(X) - : +- ColumnarExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ NoopFilter (11) - : +- ^ Scan parquet (10) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- ^ ProjectExecTransformer (29) - +- ^ NoopFilter (28) - +- ^ Scan parquet (27) + VeloxColumnarToRowExec (39) + +- TakeOrderedAndProjectExecTransformer (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + :- ^ InputIteratorTransformer (23) + : +- ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Project (63) - +- SortMergeJoin Inner (62) - :- Sort (56) - : +- Exchange (55) - : +- Project (54) - : +- SortMergeJoin Inner (53) - : :- Sort (48) - : : +- Exchange (47) - : : +- Project (46) - : : +- Filter (45) - : : +- Scan parquet (44) - : +- Sort (52) - : +- Exchange (51) - : +- Filter (50) - : +- Scan parquet (49) - +- Sort (61) - +- Exchange (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- SortMergeJoin Inner (58) + :- Sort (52) + : +- Exchange (51) + : +- Project (50) + : +- SortMergeJoin Inner (49) + : :- Sort (44) + : : +- Exchange (43) + : : +- Project (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Sort (48) + : +- Exchange (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Sort (57) + +- Exchange (56) + +- Project (55) + +- Filter (54) + +- Scan parquet (53) (1) Scan parquet @@ -94,262 +90,246 @@ Input [1]: [c_custkey#X] (8) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) SortExecTransformer -Input [1]: [c_custkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(18) SortExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(26) SortExecTransformer -Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(27) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(33) InputAdapter -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] - -(34) InputIteratorTransformer +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(35) SortExecTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(38) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(40) ProjectExecTransformer +(36) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(41) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(38) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(43) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(44) Scan parquet +(40) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(45) Filter +(41) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(46) Project +(42) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(47) Exchange +(43) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(44) Sort Input [1]: [c_custkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(49) Scan parquet +(45) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(50) Filter +(46) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(51) Exchange +(47) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(48) Sort Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(53) SortMergeJoin +(49) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(54) Project +(50) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(55) Exchange +(51) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Sort +(52) Sort Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(57) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(58) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(59) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(60) Exchange +(56) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(57) Sort Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(62) SortMergeJoin +(58) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(63) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(64) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(65) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(66) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(67) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt index e7e731e59df6..cd5f5a7dea8b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt @@ -1,51 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (53) - +- Exchange (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- SortMergeJoin LeftSemi (47) - :- Sort (41) - : +- Exchange (40) - : +- Project (39) - : +- Filter (38) - : +- Scan parquet (37) - +- Sort (46) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftSemi (45) + :- Sort (39) + : +- Exchange (38) + : +- Project (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Sort (44) + +- Exchange (43) + +- Project (42) + +- Filter (41) + +- Scan parquet (40) (1) Scan parquet @@ -81,202 +79,194 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] (8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) SortExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [1]: [l_orderkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [l_orderkey#X] -(18) SortExecTransformer -Input [1]: [l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(21) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(24) ColumnarExchange +(22) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(23) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(26) InputAdapter +(24) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(27) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(28) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(29) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) ColumnarExchange +(28) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(32) InputAdapter +(30) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(33) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(34) SortExecTransformer +(32) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(36) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(37) Scan parquet +(35) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(38) Filter +(36) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(39) Project +(37) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(40) Exchange +(38) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(39) Sort Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(42) Scan parquet +(40) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(43) Filter +(41) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(44) Project +(42) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(45) Exchange +(43) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(44) Sort Input [1]: [l_orderkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(47) SortMergeJoin +(45) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(46) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(49) HashAggregate +(47) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(50) Exchange +(48) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(49) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(52) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) Sort +(51) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(54) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt index d9ab564fef91..b0cfb4dfc2b2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt @@ -1,135 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (154) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (104) - +- ^ SortExecTransformer (102) - +- ^ InputIteratorTransformer (101) - +- ShuffleQueryStage (99), Statistics(X) - +- ColumnarExchange (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ SortExecTransformer (77) - : +- ^ InputIteratorTransformer (76) - : +- ShuffleQueryStage (74), Statistics(X) - : +- ColumnarExchange (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ SortExecTransformer (60) - : : +- ^ InputIteratorTransformer (59) - : : +- ShuffleQueryStage (57), Statistics(X) - : : +- ColumnarExchange (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ SortExecTransformer (43) - : : : +- ^ InputIteratorTransformer (42) - : : : +- ShuffleQueryStage (40), Statistics(X) - : : : +- ColumnarExchange (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : : : :- ^ SortExecTransformer (26) - : : : : +- ^ InputIteratorTransformer (25) - : : : : +- ShuffleQueryStage (23), Statistics(X) - : : : : +- ColumnarExchange (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ SortExecTransformer (9) - : : : : : +- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ SortExecTransformer (18) - : : : : +- ^ InputIteratorTransformer (17) - : : : : +- ShuffleQueryStage (15), Statistics(X) - : : : : +- ColumnarExchange (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ SortExecTransformer (35) - : : : +- ^ InputIteratorTransformer (34) - : : : +- ShuffleQueryStage (32), Statistics(X) - : : : +- ColumnarExchange (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ SortExecTransformer (52) - : : +- ^ InputIteratorTransformer (51) - : : +- ShuffleQueryStage (49), Statistics(X) - : : +- ColumnarExchange (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (153) - +- Exchange (152) - +- HashAggregate (151) - +- Exchange (150) - +- HashAggregate (149) - +- Project (148) - +- SortMergeJoin Inner (147) - :- Sort (141) - : +- Exchange (140) - : +- Project (139) - : +- SortMergeJoin Inner (138) - : :- Sort (133) - : : +- Exchange (132) - : : +- Project (131) - : : +- SortMergeJoin Inner (130) - : : :- Sort (125) - : : : +- Exchange (124) - : : : +- Project (123) - : : : +- SortMergeJoin Inner (122) - : : : :- Sort (117) - : : : : +- Exchange (116) - : : : : +- Project (115) - : : : : +- SortMergeJoin Inner (114) - : : : : :- Sort (108) - : : : : : +- Exchange (107) - : : : : : +- Filter (106) - : : : : : +- Scan parquet (105) - : : : : +- Sort (113) - : : : : +- Exchange (112) - : : : : +- Project (111) - : : : : +- Filter (110) - : : : : +- Scan parquet (109) - : : : +- Sort (121) - : : : +- Exchange (120) - : : : +- Filter (119) - : : : +- Scan parquet (118) - : : +- Sort (129) - : : +- Exchange (128) - : : +- Filter (127) - : : +- Scan parquet (126) - : +- Sort (137) - : +- Exchange (136) - : +- Filter (135) - : +- Scan parquet (134) - +- Sort (146) - +- Exchange (145) - +- Project (144) - +- Filter (143) - +- Scan parquet (142) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- SortMergeJoin Inner (137) + :- Sort (131) + : +- Exchange (130) + : +- Project (129) + : +- SortMergeJoin Inner (128) + : :- Sort (123) + : : +- Exchange (122) + : : +- Project (121) + : : +- SortMergeJoin Inner (120) + : : :- Sort (115) + : : : +- Exchange (114) + : : : +- Project (113) + : : : +- SortMergeJoin Inner (112) + : : : :- Sort (107) + : : : : +- Exchange (106) + : : : : +- Project (105) + : : : : +- SortMergeJoin Inner (104) + : : : : :- Sort (98) + : : : : : +- Exchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Sort (103) + : : : : +- Exchange (102) + : : : : +- Project (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- Sort (111) + : : : +- Exchange (110) + : : : +- Filter (109) + : : : +- Scan parquet (108) + : : +- Sort (119) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Sort (127) + : +- Exchange (126) + : +- Filter (125) + : +- Scan parquet (124) + +- Sort (136) + +- Exchange (135) + +- Project (134) + +- Filter (133) + +- Scan parquet (132) (1) Scan parquet @@ -165,618 +155,578 @@ Input [2]: [c_custkey#X, c_nationkey#X] (8) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) SortExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(18) SortExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(24) InputAdapter -Input [2]: [c_nationkey#X, o_orderkey#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(27) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(35) SortExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(41) InputAdapter -Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(42) InputIteratorTransformer +(37) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(43) SortExecTransformer +(38) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 -(44) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(50) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(51) InputIteratorTransformer +(45) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(52) SortExecTransformer +(46) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(55) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(56) ColumnarExchange +(50) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(58) InputAdapter +(52) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(59) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(60) SortExecTransformer -Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(61) Scan parquet +(54) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(62) NoopFilter +(55) NoopFilter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(60) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] - -(69) SortExecTransformer +(61) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(75) InputAdapter -Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] - -(76) InputIteratorTransformer +(67) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(77) SortExecTransformer +(68) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 -(78) Scan parquet +(69) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(79) NoopFilter +(70) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(81) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(82) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(84) InputAdapter -Input [1]: [r_regionkey#X] - -(85) InputIteratorTransformer +(75) InputAdapter Input [1]: [r_regionkey#X] -(86) SortExecTransformer +(76) InputIteratorTransformer Input [1]: [r_regionkey#X] -Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 -(87) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(88) ProjectExecTransformer +(78) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(89) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(90) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(91) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(82) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(83) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(84) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(85) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(97) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(98) ColumnarExchange +(88) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(99) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(100) InputAdapter +(90) InputAdapter Input [2]: [n_name#X, revenue#X] -(101) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(102) SortExecTransformer +(92) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(103) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(104) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(105) Scan parquet +(95) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(106) Filter +(96) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(107) Exchange +(97) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Sort +(98) Sort Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(109) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(110) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(111) Project +(101) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(112) Exchange +(102) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Sort +(103) Sort Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(114) SortMergeJoin +(104) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(115) Project +(105) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(106) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Sort +(107) Sort Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(118) Scan parquet +(108) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(119) Filter +(109) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(120) Exchange +(110) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) Sort +(111) Sort Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(122) SortMergeJoin +(112) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(123) Project +(113) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(124) Exchange +(114) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(115) Sort Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 -(126) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(127) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(128) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(119) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 -(130) SortMergeJoin +(120) SortMergeJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(131) Project +(121) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(132) Exchange +(122) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(123) Sort Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(134) Scan parquet +(124) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(135) Filter +(125) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(136) Exchange +(126) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) Sort +(127) Sort Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(138) SortMergeJoin +(128) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(139) Project +(129) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(140) Exchange +(130) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) Sort +(131) Sort Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 -(142) Scan parquet +(132) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(143) Filter +(133) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(144) Project +(134) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(145) Exchange +(135) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Sort +(136) Sort Input [1]: [r_regionkey#X] Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 -(147) SortMergeJoin +(137) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(148) Project +(138) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(149) HashAggregate +(139) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(150) Exchange +(140) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) HashAggregate +(141) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(152) Exchange +(142) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(153) Sort +(143) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(154) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt index 54caa731416b..868bb6c3a228 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt @@ -1,130 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (148) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (100) - +- ^ SortExecTransformer (98) - +- ^ InputIteratorTransformer (97) - +- ShuffleQueryStage (95), Statistics(X) - +- ColumnarExchange (94) - +- ^ RegularHashAggregateExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ ProjectExecTransformer (86) - +- ^ FlushableHashAggregateExecTransformer (85) - +- ^ ProjectExecTransformer (84) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (83) - :- ^ SortExecTransformer (77) - : +- ^ InputIteratorTransformer (76) - : +- ShuffleQueryStage (74), Statistics(X) - : +- ColumnarExchange (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ SortExecTransformer (60) - : : +- ^ InputIteratorTransformer (59) - : : +- ShuffleQueryStage (57), Statistics(X) - : : +- ColumnarExchange (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ SortExecTransformer (43) - : : : +- ^ InputIteratorTransformer (42) - : : : +- ShuffleQueryStage (40), Statistics(X) - : : : +- ColumnarExchange (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ SortExecTransformer (26) - : : : : +- ^ InputIteratorTransformer (25) - : : : : +- ShuffleQueryStage (23), Statistics(X) - : : : : +- ColumnarExchange (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ SortExecTransformer (9) - : : : : : +- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ SortExecTransformer (18) - : : : : +- ^ InputIteratorTransformer (17) - : : : : +- ShuffleQueryStage (15), Statistics(X) - : : : : +- ColumnarExchange (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ SortExecTransformer (35) - : : : +- ^ InputIteratorTransformer (34) - : : : +- ShuffleQueryStage (32), Statistics(X) - : : : +- ColumnarExchange (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ SortExecTransformer (52) - : : +- ^ InputIteratorTransformer (51) - : : +- ShuffleQueryStage (49), Statistics(X) - : : +- ColumnarExchange (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ReusedExchange (78) + VeloxColumnarToRowExec (90) + +- ^ SortExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ShuffleQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - Sort (147) - +- Exchange (146) - +- HashAggregate (145) - +- Exchange (144) - +- HashAggregate (143) - +- Project (142) - +- SortMergeJoin Inner (141) - :- Sort (136) - : +- Exchange (135) - : +- Project (134) - : +- SortMergeJoin Inner (133) - : :- Sort (128) - : : +- Exchange (127) - : : +- Project (126) - : : +- SortMergeJoin Inner (125) - : : :- Sort (120) - : : : +- Exchange (119) - : : : +- Project (118) - : : : +- SortMergeJoin Inner (117) - : : : :- Sort (112) - : : : : +- Exchange (111) - : : : : +- Project (110) - : : : : +- SortMergeJoin Inner (109) - : : : : :- Sort (104) - : : : : : +- Exchange (103) - : : : : : +- Filter (102) - : : : : : +- Scan parquet (101) - : : : : +- Sort (108) - : : : : +- Exchange (107) - : : : : +- Filter (106) - : : : : +- Scan parquet (105) - : : : +- Sort (116) - : : : +- Exchange (115) - : : : +- Filter (114) - : : : +- Scan parquet (113) - : : +- Sort (124) - : : +- Exchange (123) - : : +- Filter (122) - : : +- Scan parquet (121) - : +- Sort (132) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Sort (140) - +- Exchange (139) - +- Filter (138) - +- Scan parquet (137) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Sort (126) + : +- Exchange (125) + : +- Project (124) + : +- SortMergeJoin Inner (123) + : :- Sort (118) + : : +- Exchange (117) + : : +- Project (116) + : : +- SortMergeJoin Inner (115) + : : :- Sort (110) + : : : +- Exchange (109) + : : : +- Project (108) + : : : +- SortMergeJoin Inner (107) + : : : :- Sort (102) + : : : : +- Exchange (101) + : : : : +- Project (100) + : : : : +- SortMergeJoin Inner (99) + : : : : :- Sort (94) + : : : : : +- Exchange (93) + : : : : : +- Filter (92) + : : : : : +- Scan parquet (91) + : : : : +- Sort (98) + : : : : +- Exchange (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Sort (106) + : : : +- Exchange (105) + : : : +- Filter (104) + : : : +- Scan parquet (103) + : : +- Sort (114) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Sort (122) + : +- Exchange (121) + : +- Filter (120) + : +- Scan parquet (119) + +- Sort (130) + +- Exchange (129) + +- Filter (128) + +- Scan parquet (127) (1) Scan parquet @@ -160,590 +150,550 @@ Input [2]: [s_suppkey#X, s_nationkey#X] (8) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) SortExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(18) SortExecTransformer -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) SortExecTransformer -Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(27) Scan parquet +(24) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X, o_custkey#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(35) SortExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(41) InputAdapter +(37) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(42) InputIteratorTransformer +(38) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(43) SortExecTransformer -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(44) Scan parquet +(39) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(50) InputAdapter +(45) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(51) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(52) SortExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(55) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(56) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(58) InputAdapter +(52) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(59) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(60) SortExecTransformer -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(61) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(62) NoopFilter +(55) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(68) InputIteratorTransformer +(60) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(69) SortExecTransformer +(61) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter -Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] - -(76) InputIteratorTransformer +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(77) SortExecTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(78) ReusedExchange [Reuses operator id: 65] +(69) ReusedExchange [Reuses operator id: 58] Output [2]: [n_nationkey#X, n_name#X] -(79) ShuffleQueryStage +(70) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(80) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(81) InputIteratorTransformer +(71) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(82) SortExecTransformer +(72) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(83) ShuffledHashJoinExecTransformer +(73) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(84) ProjectExecTransformer +(74) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(85) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(86) ProjectExecTransformer +(76) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(87) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(88) ColumnarExchange +(78) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(79) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(90) InputAdapter +(80) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(91) InputIteratorTransformer +(81) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(92) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(93) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(94) ColumnarExchange +(84) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(95) ShuffleQueryStage +(85) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(96) InputAdapter +(86) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(97) InputIteratorTransformer +(87) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(98) SortExecTransformer +(88) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(99) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(100) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(101) Scan parquet +(91) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(102) Filter +(92) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(103) Exchange +(93) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Sort +(94) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(105) Scan parquet +(95) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(106) Filter +(96) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(107) Exchange +(97) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Sort +(98) Sort Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(109) SortMergeJoin +(99) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(110) Project +(100) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(111) Exchange +(101) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Sort +(102) Sort Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(113) Scan parquet +(103) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(114) Filter +(104) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(115) Exchange +(105) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Sort +(106) Sort Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(117) SortMergeJoin +(107) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(118) Project +(108) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(119) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(110) Sort Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(121) Scan parquet +(111) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(122) Filter +(112) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(123) Exchange +(113) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) Sort +(114) Sort Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(125) SortMergeJoin +(115) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(126) Project +(116) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(127) Exchange +(117) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(118) Sort Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(129) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(130) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(131) Exchange +(121) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(122) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(133) SortMergeJoin +(123) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(134) Project +(124) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(135) Exchange +(125) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Sort +(126) Sort Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(137) Scan parquet +(127) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(138) Filter +(128) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(139) Exchange +(129) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(140) Sort +(130) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(141) SortMergeJoin +(131) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(142) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(143) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(144) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(146) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(147) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(148) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt index 2af4c574ce31..d1d7c76f131c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt @@ -1,178 +1,164 @@ == Physical Plan == -AdaptiveSparkPlan (205) +AdaptiveSparkPlan (191) +- == Final Plan == - VeloxColumnarToRowExec (139) - +- ^ SortExecTransformer (137) - +- ^ InputIteratorTransformer (136) - +- ShuffleQueryStage (134), Statistics(X) - +- ColumnarExchange (133) - +- ^ ProjectExecTransformer (131) - +- ^ RegularHashAggregateExecTransformer (130) - +- ^ InputIteratorTransformer (129) - +- ShuffleQueryStage (127), Statistics(X) - +- ColumnarExchange (126) - +- ^ ProjectExecTransformer (124) - +- ^ FlushableHashAggregateExecTransformer (123) - +- ^ ProjectExecTransformer (122) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) - :- ^ SortExecTransformer (111) - : +- ^ InputIteratorTransformer (110) - : +- ShuffleQueryStage (108), Statistics(X) - : +- ColumnarExchange (107) - : +- ^ ProjectExecTransformer (105) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - : :- ^ SortExecTransformer (94) - : : +- ^ InputIteratorTransformer (93) - : : +- ShuffleQueryStage (91), Statistics(X) - : : +- ColumnarExchange (90) - : : +- ^ ProjectExecTransformer (88) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - : : :- ^ SortExecTransformer (77) - : : : +- ^ InputIteratorTransformer (76) - : : : +- ShuffleQueryStage (74), Statistics(X) - : : : +- ColumnarExchange (73) - : : : +- ^ ProjectExecTransformer (71) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : : : :- ^ SortExecTransformer (60) - : : : : +- ^ InputIteratorTransformer (59) - : : : : +- ShuffleQueryStage (57), Statistics(X) - : : : : +- ColumnarExchange (56) - : : : : +- ^ ProjectExecTransformer (54) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : : : :- ^ SortExecTransformer (43) - : : : : : +- ^ InputIteratorTransformer (42) - : : : : : +- ShuffleQueryStage (40), Statistics(X) - : : : : : +- ColumnarExchange (39) - : : : : : +- ^ ProjectExecTransformer (37) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : : : :- ^ SortExecTransformer (26) - : : : : : : +- ^ InputIteratorTransformer (25) - : : : : : : +- ShuffleQueryStage (23), Statistics(X) - : : : : : : +- ColumnarExchange (22) - : : : : : : +- ^ ProjectExecTransformer (20) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : : : :- ^ SortExecTransformer (9) - : : : : : : : +- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ SortExecTransformer (18) - : : : : : : +- ^ InputIteratorTransformer (17) - : : : : : : +- ShuffleQueryStage (15), Statistics(X) - : : : : : : +- ColumnarExchange (14) - : : : : : : +- ^ ProjectExecTransformer (12) - : : : : : : +- ^ NoopFilter (11) - : : : : : : +- ^ Scan parquet (10) - : : : : : +- ^ SortExecTransformer (35) - : : : : : +- ^ InputIteratorTransformer (34) - : : : : : +- ShuffleQueryStage (32), Statistics(X) - : : : : : +- ColumnarExchange (31) - : : : : : +- ^ ProjectExecTransformer (29) - : : : : : +- ^ NoopFilter (28) - : : : : : +- ^ Scan parquet (27) - : : : : +- ^ SortExecTransformer (52) - : : : : +- ^ InputIteratorTransformer (51) - : : : : +- ShuffleQueryStage (49), Statistics(X) - : : : : +- ColumnarExchange (48) - : : : : +- ^ ProjectExecTransformer (46) - : : : : +- ^ NoopFilter (45) - : : : : +- ^ Scan parquet (44) - : : : +- ^ SortExecTransformer (69) - : : : +- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ NoopFilter (62) - : : : +- ^ Scan parquet (61) - : : +- ^ SortExecTransformer (86) - : : +- ^ InputIteratorTransformer (85) - : : +- ShuffleQueryStage (83), Statistics(X) - : : +- ColumnarExchange (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ NoopFilter (79) - : : +- ^ Scan parquet (78) - : +- ^ SortExecTransformer (103) - : +- ^ InputIteratorTransformer (102) - : +- ShuffleQueryStage (100), Statistics(X) - : +- ColumnarExchange (99) - : +- ^ ProjectExecTransformer (97) - : +- ^ NoopFilter (96) - : +- ^ Scan parquet (95) - +- ^ SortExecTransformer (120) - +- ^ InputIteratorTransformer (119) - +- ShuffleQueryStage (117), Statistics(X) - +- ColumnarExchange (116) - +- ^ ProjectExecTransformer (114) - +- ^ NoopFilter (113) - +- ^ Scan parquet (112) + VeloxColumnarToRowExec (125) + +- ^ SortExecTransformer (123) + +- ^ InputIteratorTransformer (122) + +- ShuffleQueryStage (120), Statistics(X) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ShuffleQueryStage (113), Statistics(X) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (83) + : : +- ShuffleQueryStage (81), Statistics(X) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ShuffleQueryStage (66), Statistics(X) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ShuffleQueryStage (51), Statistics(X) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ShuffleQueryStage (36), Statistics(X) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ShuffleQueryStage (29), Statistics(X) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ NoopFilter (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ShuffleQueryStage (44), Statistics(X) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ NoopFilter (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ShuffleQueryStage (59), Statistics(X) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ NoopFilter (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ShuffleQueryStage (74), Statistics(X) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ NoopFilter (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ NoopFilter (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ShuffleQueryStage (104), Statistics(X) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ NoopFilter (100) + +- ^ Scan parquet (99) +- == Initial Plan == - Sort (204) - +- Exchange (203) - +- HashAggregate (202) - +- Exchange (201) - +- HashAggregate (200) - +- Project (199) - +- SortMergeJoin Inner (198) - :- Sort (192) - : +- Exchange (191) - : +- Project (190) - : +- SortMergeJoin Inner (189) - : :- Sort (184) - : : +- Exchange (183) - : : +- Project (182) - : : +- SortMergeJoin Inner (181) - : : :- Sort (176) - : : : +- Exchange (175) - : : : +- Project (174) - : : : +- SortMergeJoin Inner (173) - : : : :- Sort (168) - : : : : +- Exchange (167) - : : : : +- Project (166) - : : : : +- SortMergeJoin Inner (165) - : : : : :- Sort (160) - : : : : : +- Exchange (159) - : : : : : +- Project (158) - : : : : : +- SortMergeJoin Inner (157) - : : : : : :- Sort (152) - : : : : : : +- Exchange (151) - : : : : : : +- Project (150) - : : : : : : +- SortMergeJoin Inner (149) - : : : : : : :- Sort (144) - : : : : : : : +- Exchange (143) - : : : : : : : +- Project (142) - : : : : : : : +- Filter (141) - : : : : : : : +- Scan parquet (140) - : : : : : : +- Sort (148) - : : : : : : +- Exchange (147) - : : : : : : +- Filter (146) - : : : : : : +- Scan parquet (145) - : : : : : +- Sort (156) - : : : : : +- Exchange (155) - : : : : : +- Filter (154) - : : : : : +- Scan parquet (153) - : : : : +- Sort (164) - : : : : +- Exchange (163) - : : : : +- Filter (162) - : : : : +- Scan parquet (161) - : : : +- Sort (172) - : : : +- Exchange (171) - : : : +- Filter (170) - : : : +- Scan parquet (169) - : : +- Sort (180) - : : +- Exchange (179) - : : +- Filter (178) - : : +- Scan parquet (177) - : +- Sort (188) - : +- Exchange (187) - : +- Filter (186) - : +- Scan parquet (185) - +- Sort (197) - +- Exchange (196) - +- Project (195) - +- Filter (194) - +- Scan parquet (193) + Sort (190) + +- Exchange (189) + +- HashAggregate (188) + +- Exchange (187) + +- HashAggregate (186) + +- Project (185) + +- SortMergeJoin Inner (184) + :- Sort (178) + : +- Exchange (177) + : +- Project (176) + : +- SortMergeJoin Inner (175) + : :- Sort (170) + : : +- Exchange (169) + : : +- Project (168) + : : +- SortMergeJoin Inner (167) + : : :- Sort (162) + : : : +- Exchange (161) + : : : +- Project (160) + : : : +- SortMergeJoin Inner (159) + : : : :- Sort (154) + : : : : +- Exchange (153) + : : : : +- Project (152) + : : : : +- SortMergeJoin Inner (151) + : : : : :- Sort (146) + : : : : : +- Exchange (145) + : : : : : +- Project (144) + : : : : : +- SortMergeJoin Inner (143) + : : : : : :- Sort (138) + : : : : : : +- Exchange (137) + : : : : : : +- Project (136) + : : : : : : +- SortMergeJoin Inner (135) + : : : : : : :- Sort (130) + : : : : : : : +- Exchange (129) + : : : : : : : +- Project (128) + : : : : : : : +- Filter (127) + : : : : : : : +- Scan parquet (126) + : : : : : : +- Sort (134) + : : : : : : +- Exchange (133) + : : : : : : +- Filter (132) + : : : : : : +- Scan parquet (131) + : : : : : +- Sort (142) + : : : : : +- Exchange (141) + : : : : : +- Filter (140) + : : : : : +- Scan parquet (139) + : : : : +- Sort (150) + : : : : +- Exchange (149) + : : : : +- Filter (148) + : : : : +- Scan parquet (147) + : : : +- Sort (158) + : : : +- Exchange (157) + : : : +- Filter (156) + : : : +- Scan parquet (155) + : : +- Sort (166) + : : +- Exchange (165) + : : +- Filter (164) + : : +- Scan parquet (163) + : +- Sort (174) + : +- Exchange (173) + : +- Filter (172) + : +- Scan parquet (171) + +- Sort (183) + +- Exchange (182) + +- Project (181) + +- Filter (180) + +- Scan parquet (179) (1) Scan parquet @@ -208,830 +194,774 @@ Input [1]: [p_partkey#X] (8) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) SortExecTransformer -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(18) SortExecTransformer -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(24) InputAdapter -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(27) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(35) SortExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(41) InputAdapter -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(42) InputIteratorTransformer +(37) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(43) SortExecTransformer +(38) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(44) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(50) InputAdapter -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(51) InputIteratorTransformer +(45) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(52) SortExecTransformer +(46) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(55) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(56) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(58) InputAdapter +(52) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(59) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(60) SortExecTransformer -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(61) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) NoopFilter +(55) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(60) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(68) InputIteratorTransformer -Input [2]: [c_custkey#X, c_nationkey#X] - -(69) SortExecTransformer +(61) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(75) InputAdapter -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] - -(76) InputIteratorTransformer +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(77) SortExecTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(78) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(79) NoopFilter +(70) NoopFilter Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X, n_regionkey#X] -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(81) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(82) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(84) InputAdapter -Input [2]: [n_nationkey#X, n_regionkey#X] - -(85) InputIteratorTransformer +(75) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(86) SortExecTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(87) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) ProjectExecTransformer +(78) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(89) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(90) ColumnarExchange +(80) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(91) ShuffleQueryStage +(81) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(92) InputAdapter +(82) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(93) InputIteratorTransformer +(83) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(94) SortExecTransformer -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(95) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(96) NoopFilter +(85) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(97) ProjectExecTransformer +(86) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(98) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(99) ColumnarExchange +(88) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(101) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(102) InputIteratorTransformer +(90) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(103) SortExecTransformer +(91) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(104) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(105) ProjectExecTransformer +(93) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(106) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(107) ColumnarExchange +(95) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(108) ShuffleQueryStage +(96) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(109) InputAdapter +(97) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(110) InputIteratorTransformer +(98) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(111) SortExecTransformer -Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 - -(112) Scan parquet +(99) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(113) NoopFilter +(100) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(114) ProjectExecTransformer +(101) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(115) WholeStageCodegenTransformer (X) +(102) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(116) ColumnarExchange +(103) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(117) ShuffleQueryStage +(104) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(118) InputAdapter -Input [1]: [r_regionkey#X] - -(119) InputIteratorTransformer +(105) InputAdapter Input [1]: [r_regionkey#X] -(120) SortExecTransformer +(106) InputIteratorTransformer Input [1]: [r_regionkey#X] -Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 -(121) ShuffledHashJoinExecTransformer +(107) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(122) ProjectExecTransformer +(108) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(123) FlushableHashAggregateExecTransformer +(109) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(124) ProjectExecTransformer +(110) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) WholeStageCodegenTransformer (X) +(111) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(126) ColumnarExchange +(112) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(127) ShuffleQueryStage +(113) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(128) InputAdapter +(114) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(129) InputIteratorTransformer +(115) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(130) RegularHashAggregateExecTransformer +(116) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(131) ProjectExecTransformer +(117) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(132) WholeStageCodegenTransformer (X) +(118) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(133) ColumnarExchange +(119) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(134) ShuffleQueryStage +(120) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(135) InputAdapter +(121) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(136) InputIteratorTransformer +(122) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(137) SortExecTransformer +(123) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(138) WholeStageCodegenTransformer (X) +(124) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(139) VeloxColumnarToRowExec +(125) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(140) Scan parquet +(126) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(141) Filter +(127) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(142) Project +(128) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(143) Exchange +(129) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(130) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(145) Scan parquet +(131) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(146) Filter +(132) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(147) Exchange +(133) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Sort +(134) Sort Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(149) SortMergeJoin +(135) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(150) Project +(136) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(151) Exchange +(137) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Sort +(138) Sort Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(153) Scan parquet +(139) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(154) Filter +(140) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(155) Exchange +(141) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(156) Sort +(142) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(157) SortMergeJoin +(143) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(158) Project +(144) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(159) Exchange +(145) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Sort +(146) Sort Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(161) Scan parquet +(147) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(162) Filter +(148) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(163) Exchange +(149) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(164) Sort +(150) Sort Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(165) SortMergeJoin +(151) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(166) Project +(152) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(167) Exchange +(153) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(168) Sort +(154) Sort Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(169) Scan parquet +(155) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(170) Filter +(156) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(171) Exchange +(157) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(172) Sort +(158) Sort Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(173) SortMergeJoin +(159) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(174) Project +(160) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(175) Exchange +(161) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(162) Sort Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(177) Scan parquet +(163) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(178) Filter +(164) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(179) Exchange +(165) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(180) Sort +(166) Sort Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(181) SortMergeJoin +(167) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(182) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(183) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(184) Sort +(170) Sort Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(185) Scan parquet +(171) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(186) Filter +(172) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(187) Exchange +(173) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(188) Sort +(174) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(189) SortMergeJoin +(175) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(190) Project +(176) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(191) Exchange +(177) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(192) Sort +(178) Sort Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 -(193) Scan parquet +(179) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(194) Filter +(180) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(195) Project +(181) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(196) Exchange +(182) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(197) Sort +(183) Sort Input [1]: [r_regionkey#X] Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 -(198) SortMergeJoin +(184) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(199) Project +(185) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(200) HashAggregate +(186) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(201) Exchange +(187) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(202) HashAggregate +(188) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(203) Exchange +(189) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(204) Sort +(190) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(205) AdaptiveSparkPlan +(191) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt index f17d2c243dc2..e8e7a4611f62 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt @@ -1,134 +1,124 @@ == Physical Plan == -AdaptiveSparkPlan (153) +AdaptiveSparkPlan (143) +- == Final Plan == - VeloxColumnarToRowExec (104) - +- ^ SortExecTransformer (102) - +- ^ InputIteratorTransformer (101) - +- ShuffleQueryStage (99), Statistics(X) - +- ColumnarExchange (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ SortExecTransformer (77) - : +- ^ InputIteratorTransformer (76) - : +- ShuffleQueryStage (74), Statistics(X) - : +- ColumnarExchange (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : :- ^ SortExecTransformer (60) - : : +- ^ InputIteratorTransformer (59) - : : +- ShuffleQueryStage (57), Statistics(X) - : : +- ColumnarExchange (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : :- ^ SortExecTransformer (43) - : : : +- ^ InputIteratorTransformer (42) - : : : +- ShuffleQueryStage (40), Statistics(X) - : : : +- ColumnarExchange (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ SortExecTransformer (26) - : : : : +- ^ InputIteratorTransformer (25) - : : : : +- ShuffleQueryStage (23), Statistics(X) - : : : : +- ColumnarExchange (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ SortExecTransformer (9) - : : : : : +- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ SortExecTransformer (18) - : : : : +- ^ InputIteratorTransformer (17) - : : : : +- ShuffleQueryStage (15), Statistics(X) - : : : : +- ColumnarExchange (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ SortExecTransformer (35) - : : : +- ^ InputIteratorTransformer (34) - : : : +- ShuffleQueryStage (32), Statistics(X) - : : : +- ColumnarExchange (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ SortExecTransformer (52) - : : +- ^ InputIteratorTransformer (51) - : : +- ShuffleQueryStage (49), Statistics(X) - : : +- ColumnarExchange (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (152) - +- Exchange (151) - +- HashAggregate (150) - +- Exchange (149) - +- HashAggregate (148) - +- Project (147) - +- SortMergeJoin Inner (146) - :- Sort (141) - : +- Exchange (140) - : +- Project (139) - : +- SortMergeJoin Inner (138) - : :- Sort (133) - : : +- Exchange (132) - : : +- Project (131) - : : +- SortMergeJoin Inner (130) - : : :- Sort (125) - : : : +- Exchange (124) - : : : +- Project (123) - : : : +- SortMergeJoin Inner (122) - : : : :- Sort (117) - : : : : +- Exchange (116) - : : : : +- Project (115) - : : : : +- SortMergeJoin Inner (114) - : : : : :- Sort (109) - : : : : : +- Exchange (108) - : : : : : +- Project (107) - : : : : : +- Filter (106) - : : : : : +- Scan parquet (105) - : : : : +- Sort (113) - : : : : +- Exchange (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Sort (121) - : : : +- Exchange (120) - : : : +- Filter (119) - : : : +- Scan parquet (118) - : : +- Sort (129) - : : +- Exchange (128) - : : +- Filter (127) - : : +- Scan parquet (126) - : +- Sort (137) - : +- Exchange (136) - : +- Filter (135) - : +- Scan parquet (134) - +- Sort (145) - +- Exchange (144) - +- Filter (143) - +- Scan parquet (142) + Sort (142) + +- Exchange (141) + +- HashAggregate (140) + +- Exchange (139) + +- HashAggregate (138) + +- Project (137) + +- SortMergeJoin Inner (136) + :- Sort (131) + : +- Exchange (130) + : +- Project (129) + : +- SortMergeJoin Inner (128) + : :- Sort (123) + : : +- Exchange (122) + : : +- Project (121) + : : +- SortMergeJoin Inner (120) + : : :- Sort (115) + : : : +- Exchange (114) + : : : +- Project (113) + : : : +- SortMergeJoin Inner (112) + : : : :- Sort (107) + : : : : +- Exchange (106) + : : : : +- Project (105) + : : : : +- SortMergeJoin Inner (104) + : : : : :- Sort (99) + : : : : : +- Exchange (98) + : : : : : +- Project (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Sort (103) + : : : : +- Exchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- Sort (111) + : : : +- Exchange (110) + : : : +- Filter (109) + : : : +- Scan parquet (108) + : : +- Sort (119) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Sort (127) + : +- Exchange (126) + : +- Filter (125) + : +- Scan parquet (124) + +- Sort (135) + +- Exchange (134) + +- Filter (133) + +- Scan parquet (132) (1) Scan parquet @@ -164,614 +154,574 @@ Input [1]: [p_partkey#X] (8) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) SortExecTransformer -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(18) SortExecTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(24) InputAdapter -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(27) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(35) SortExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(41) InputAdapter -Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(42) InputIteratorTransformer +(37) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(43) SortExecTransformer +(38) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 -(44) Scan parquet +(39) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(50) InputAdapter +(45) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(51) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(52) SortExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 - -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(55) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(56) ColumnarExchange +(50) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(58) InputAdapter -Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] - -(59) InputIteratorTransformer +(52) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(60) SortExecTransformer +(53) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(61) Scan parquet +(54) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(62) NoopFilter +(55) NoopFilter Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_orderdate#X] -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(67) InputAdapter -Input [2]: [o_orderkey#X, o_orderdate#X] - -(68) InputIteratorTransformer +(60) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(69) SortExecTransformer +(61) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(75) InputAdapter -Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] - -(76) InputIteratorTransformer +(67) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(77) SortExecTransformer +(68) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(78) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) NoopFilter +(70) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(81) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(82) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(84) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(85) InputIteratorTransformer +(75) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(86) SortExecTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(87) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) ProjectExecTransformer +(78) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(89) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(90) ProjectExecTransformer +(80) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(91) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(82) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(83) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(84) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(85) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(98) ColumnarExchange +(88) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(99) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(100) InputAdapter +(90) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(101) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(102) SortExecTransformer +(92) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(103) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(104) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(105) Scan parquet +(95) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(106) Filter +(96) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(107) Project +(97) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(108) Exchange +(98) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Sort +(99) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(110) Scan parquet +(100) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(111) Filter +(101) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(112) Exchange +(102) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Sort +(103) Sort Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(114) SortMergeJoin +(104) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(115) Project +(105) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(116) Exchange +(106) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Sort +(107) Sort Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(118) Scan parquet +(108) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(119) Filter +(109) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(120) Exchange +(110) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) Sort +(111) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(122) SortMergeJoin +(112) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(123) Project +(113) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(124) Exchange +(114) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(115) Sort Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 -(126) Scan parquet +(116) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(127) Filter +(117) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(128) Exchange +(118) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(119) Sort Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 -(130) SortMergeJoin +(120) SortMergeJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(131) Project +(121) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(132) Exchange +(122) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(123) Sort Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(134) Scan parquet +(124) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(135) Filter +(125) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(136) Exchange +(126) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) Sort +(127) Sort Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(138) SortMergeJoin +(128) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(139) Project +(129) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(140) Exchange +(130) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) Sort +(131) Sort Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(142) Scan parquet +(132) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(143) Filter +(133) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(144) Exchange +(134) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Sort +(135) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(146) SortMergeJoin +(136) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(147) Project +(137) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(148) HashAggregate +(138) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(149) Exchange +(139) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(150) HashAggregate +(140) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(151) Exchange +(141) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Sort +(142) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(153) AdaptiveSparkPlan +(143) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt index 57debca4533c..6889b17f2406 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt @@ -1,90 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- TakeOrderedAndProjectExecTransformer (65) - +- ^ ProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ ProjectExecTransformer (56) - +- ^ FlushableHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - :- ^ SortExecTransformer (43) - : +- ^ InputIteratorTransformer (42) - : +- ShuffleQueryStage (40), Statistics(X) - : +- ColumnarExchange (39) - : +- ^ ProjectExecTransformer (37) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : :- ^ SortExecTransformer (26) - : : +- ^ InputIteratorTransformer (25) - : : +- ShuffleQueryStage (23), Statistics(X) - : : +- ColumnarExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : : :- ^ SortExecTransformer (9) - : : : +- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ SortExecTransformer (18) - : : +- ^ InputIteratorTransformer (17) - : : +- ShuffleQueryStage (15), Statistics(X) - : : +- ColumnarExchange (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ NoopFilter (11) - : : +- ^ Scan parquet (10) - : +- ^ SortExecTransformer (35) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ NoopFilter (28) - : +- ^ Scan parquet (27) - +- ^ SortExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ NoopFilter (45) - +- ^ Scan parquet (44) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + :- ^ InputIteratorTransformer (38) + : +- ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : :- ^ InputIteratorTransformer (23) + : : +- ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- SortMergeJoin Inner (93) - :- Sort (88) - : +- Exchange (87) - : +- Project (86) - : +- SortMergeJoin Inner (85) - : :- Sort (79) - : : +- Exchange (78) - : : +- Project (77) - : : +- SortMergeJoin Inner (76) - : : :- Sort (70) - : : : +- Exchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- Sort (75) - : : +- Exchange (74) - : : +- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Sort (84) - : +- Exchange (83) - : +- Project (82) - : +- Filter (81) - : +- Scan parquet (80) - +- Sort (92) - +- Exchange (91) - +- Filter (90) - +- Scan parquet (89) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- SortMergeJoin Inner (87) + :- Sort (82) + : +- Exchange (81) + : +- Project (80) + : +- SortMergeJoin Inner (79) + : :- Sort (73) + : : +- Exchange (72) + : : +- Project (71) + : : +- SortMergeJoin Inner (70) + : : :- Sort (64) + : : : +- Exchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- Sort (69) + : : +- Exchange (68) + : : +- Project (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Sort (78) + : +- Exchange (77) + : +- Project (76) + : +- Filter (75) + : +- Scan parquet (74) + +- Sort (86) + +- Exchange (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -120,398 +114,374 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct (8) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) SortExecTransformer -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(18) SortExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(24) InputAdapter -Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(27) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(35) SortExecTransformer -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(41) InputAdapter +(37) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(42) InputIteratorTransformer +(38) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(43) SortExecTransformer -Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 - -(44) Scan parquet +(39) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(50) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(51) InputIteratorTransformer +(45) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(52) SortExecTransformer +(46) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(55) FlushableHashAggregateExecTransformer +(49) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) ProjectExecTransformer +(50) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(57) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(58) ColumnarExchange +(52) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(53) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(60) InputAdapter +(54) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(61) InputIteratorTransformer +(55) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(62) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(63) ProjectExecTransformer +(57) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(64) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(65) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(66) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) Scan parquet +(61) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(62) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) Exchange +(63) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) Sort +(64) Sort Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(71) Scan parquet +(65) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(72) Filter +(66) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(73) Project +(67) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(74) Exchange +(68) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) Sort +(69) Sort Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(76) SortMergeJoin +(70) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(77) Project +(71) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(78) Exchange +(72) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Sort +(73) Sort Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(80) Scan parquet +(74) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(81) Filter +(75) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(82) Project +(76) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(83) Exchange +(77) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Sort +(78) Sort Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(85) SortMergeJoin +(79) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(86) Project +(80) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(87) Exchange +(81) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Sort +(82) Sort Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(89) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(90) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(91) Exchange +(85) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(86) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(93) SortMergeJoin +(87) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(94) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(95) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(96) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(98) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(99) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt index 686de2effa9d..3eddaf71d8a5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt @@ -1,73 +1,69 @@ == Physical Plan == -AdaptiveSparkPlan (80) +AdaptiveSparkPlan (76) +- == Final Plan == - VeloxColumnarToRowExec (54) - +- ^ SortExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ FilterExecTransformer (46) - +- ^ RegularHashAggregateExecTransformer (45) - +- ^ InputIteratorTransformer (44) - +- ShuffleQueryStage (42), Statistics(X) - +- ColumnarExchange (41) - +- ^ ProjectExecTransformer (39) - +- ^ FlushableHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - :- ^ SortExecTransformer (26) - : +- ^ InputIteratorTransformer (25) - : +- ShuffleQueryStage (23), Statistics(X) - : +- ColumnarExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ NoopFilter (11) - : +- ^ Scan parquet (10) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- ^ ProjectExecTransformer (29) - +- ^ NoopFilter (28) - +- ^ Scan parquet (27) + VeloxColumnarToRowExec (50) + +- ^ SortExecTransformer (48) + +- ^ InputIteratorTransformer (47) + +- ShuffleQueryStage (45), Statistics(X) + +- ColumnarExchange (44) + +- ^ FilterExecTransformer (42) + +- ^ RegularHashAggregateExecTransformer (41) + +- ^ InputIteratorTransformer (40) + +- ShuffleQueryStage (38), Statistics(X) + +- ColumnarExchange (37) + +- ^ ProjectExecTransformer (35) + +- ^ FlushableHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + :- ^ InputIteratorTransformer (23) + : +- ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - Sort (79) - +- Exchange (78) - +- Filter (77) - +- HashAggregate (76) - +- Exchange (75) - +- HashAggregate (74) - +- Project (73) - +- SortMergeJoin Inner (72) - :- Sort (66) - : +- Exchange (65) - : +- Project (64) - : +- SortMergeJoin Inner (63) - : :- Sort (58) - : : +- Exchange (57) - : : +- Filter (56) - : : +- Scan parquet (55) - : +- Sort (62) - : +- Exchange (61) - : +- Filter (60) - : +- Scan parquet (59) - +- Sort (71) - +- Exchange (70) - +- Project (69) - +- Filter (68) - +- Scan parquet (67) + Sort (75) + +- Exchange (74) + +- Filter (73) + +- HashAggregate (72) + +- Exchange (71) + +- HashAggregate (70) + +- Project (69) + +- SortMergeJoin Inner (68) + :- Sort (62) + : +- Exchange (61) + : +- Project (60) + : +- SortMergeJoin Inner (59) + : :- Sort (54) + : : +- Exchange (53) + : : +- Filter (52) + : : +- Scan parquet (51) + : +- Sort (58) + : +- Exchange (57) + : +- Filter (56) + : +- Scan parquet (55) + +- Sort (67) + +- Exchange (66) + +- Project (65) + +- Filter (64) + +- Scan parquet (63) (1) Scan parquet @@ -103,615 +99,579 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] (8) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) SortExecTransformer -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(18) SortExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(26) SortExecTransformer -Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(27) Scan parquet +(24) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(33) InputAdapter -Input [1]: [n_nationkey#X] - -(34) InputIteratorTransformer +(30) InputAdapter Input [1]: [n_nationkey#X] -(35) SortExecTransformer +(31) InputIteratorTransformer Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(38) FlushableHashAggregateExecTransformer +(34) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(39) ProjectExecTransformer +(35) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(41) ColumnarExchange +(37) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(42) ShuffleQueryStage +(38) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(43) InputAdapter +(39) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(44) InputIteratorTransformer +(40) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(45) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(46) FilterExecTransformer +(42) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(47) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(48) ColumnarExchange +(44) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(45) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(50) InputAdapter +(46) InputAdapter Input [2]: [ps_partkey#X, value#X] -(51) InputIteratorTransformer +(47) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(52) SortExecTransformer +(48) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(53) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(54) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(55) Scan parquet +(51) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(56) Filter +(52) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(57) Exchange +(53) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) Sort +(54) Sort Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 -(59) Scan parquet +(55) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(56) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) Exchange +(57) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) Sort +(58) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(63) SortMergeJoin +(59) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(64) Project +(60) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(65) Exchange +(61) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) Sort +(62) Sort Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(67) Scan parquet +(63) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(68) Filter +(64) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(69) Project +(65) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(70) Exchange +(66) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(67) Sort Input [1]: [n_nationkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(72) SortMergeJoin +(68) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(73) Project +(69) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(74) HashAggregate +(70) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(75) Exchange +(71) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) HashAggregate +(72) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(77) Filter +(73) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(78) Exchange +(74) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Sort +(75) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(80) AdaptiveSparkPlan +(76) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (136) +Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (114) - +- ^ ProjectExecTransformer (112) - +- ^ RegularHashAggregateExecTransformer (111) - +- ^ RegularHashAggregateExecTransformer (110) - +- ^ ProjectExecTransformer (109) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (108) - :- ^ SortExecTransformer (102) - : +- ^ InputIteratorTransformer (101) - : +- ShuffleQueryStage (99), Statistics(X) - : +- ColumnarExchange (98) - : +- ^ ProjectExecTransformer (96) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (95) - : :- ^ SortExecTransformer (89) - : : +- ^ InputIteratorTransformer (88) - : : +- ShuffleQueryStage (86), Statistics(X) - : : +- ColumnarExchange (85) - : : +- ^ ProjectExecTransformer (83) - : : +- ^ NoopFilter (82) - : : +- ^ Scan parquet (81) - : +- ^ SortExecTransformer (94) - : +- ^ InputIteratorTransformer (93) - : +- ShuffleQueryStage (91), Statistics(X) - : +- ReusedExchange (90) - +- ^ SortExecTransformer (107) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ReusedExchange (103) + VeloxColumnarToRowExec (106) + +- ^ ProjectExecTransformer (104) + +- ^ RegularHashAggregateExecTransformer (103) + +- ^ RegularHashAggregateExecTransformer (102) + +- ^ ProjectExecTransformer (101) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (100) + :- ^ InputIteratorTransformer (95) + : +- ShuffleQueryStage (93), Statistics(X) + : +- ColumnarExchange (92) + : +- ^ ProjectExecTransformer (90) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (89) + : :- ^ InputIteratorTransformer (84) + : : +- ShuffleQueryStage (82), Statistics(X) + : : +- ColumnarExchange (81) + : : +- ^ ProjectExecTransformer (79) + : : +- ^ NoopFilter (78) + : : +- ^ Scan parquet (77) + : +- ^ InputIteratorTransformer (88) + : +- ShuffleQueryStage (86), Statistics(X) + : +- ReusedExchange (85) + +- ^ InputIteratorTransformer (99) + +- ShuffleQueryStage (97), Statistics(X) + +- ReusedExchange (96) +- == Initial Plan == - HashAggregate (135) - +- HashAggregate (134) - +- Project (133) - +- SortMergeJoin Inner (132) - :- Sort (126) - : +- Exchange (125) - : +- Project (124) - : +- SortMergeJoin Inner (123) - : :- Sort (118) - : : +- Exchange (117) - : : +- Filter (116) - : : +- Scan parquet (115) - : +- Sort (122) - : +- Exchange (121) - : +- Filter (120) - : +- Scan parquet (119) - +- Sort (131) - +- Exchange (130) - +- Project (129) - +- Filter (128) - +- Scan parquet (127) - - -(81) Scan parquet + HashAggregate (127) + +- HashAggregate (126) + +- Project (125) + +- SortMergeJoin Inner (124) + :- Sort (118) + : +- Exchange (117) + : +- Project (116) + : +- SortMergeJoin Inner (115) + : :- Sort (110) + : : +- Exchange (109) + : : +- Filter (108) + : : +- Scan parquet (107) + : +- Sort (114) + : +- Exchange (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Sort (123) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) + + +(77) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(82) NoopFilter +(78) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(83) ProjectExecTransformer +(79) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(84) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(85) ColumnarExchange +(81) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(86) ShuffleQueryStage +(82) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(87) InputAdapter +(83) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(88) InputIteratorTransformer +(84) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(89) SortExecTransformer -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 - -(90) ReusedExchange [Reuses operator id: 14] +(85) ReusedExchange [Reuses operator id: 13] Output [2]: [s_suppkey#X, s_nationkey#X] -(91) ShuffleQueryStage +(86) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(92) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(93) InputIteratorTransformer +(87) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(94) SortExecTransformer +(88) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(95) ShuffledHashJoinExecTransformer +(89) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(96) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(97) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(98) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(99) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(100) InputAdapter -Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] - -(101) InputIteratorTransformer +(94) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(102) SortExecTransformer +(95) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(103) ReusedExchange [Reuses operator id: 31] +(96) ReusedExchange [Reuses operator id: 28] Output [1]: [n_nationkey#X] -(104) ShuffleQueryStage +(97) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(105) InputAdapter +(98) InputAdapter Input [1]: [n_nationkey#X] -(106) InputIteratorTransformer +(99) InputIteratorTransformer Input [1]: [n_nationkey#X] -(107) SortExecTransformer -Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(108) ShuffledHashJoinExecTransformer +(100) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(109) ProjectExecTransformer +(101) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(110) RegularHashAggregateExecTransformer +(102) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(111) RegularHashAggregateExecTransformer +(103) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(112) ProjectExecTransformer +(104) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(113) WholeStageCodegenTransformer (X) +(105) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(114) VeloxColumnarToRowExec +(106) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(115) Scan parquet +(107) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(116) Filter +(108) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(117) Exchange +(109) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Sort +(110) Sort Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 -(119) Scan parquet +(111) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(120) Filter +(112) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(121) Exchange +(113) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(114) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(123) SortMergeJoin +(115) SortMergeJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(124) Project +(116) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(125) Exchange +(117) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) Sort +(118) Sort Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(127) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(128) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(129) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(130) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) Sort +(123) Sort Input [1]: [n_nationkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(132) SortMergeJoin +(124) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(133) Project +(125) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(134) HashAggregate +(126) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(135) HashAggregate +(127) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(136) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt index 6d4a080c59c4..63f925581ca0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt @@ -1,50 +1,48 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (51) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (52) - +- Exchange (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- SortMergeJoin Inner (46) - :- Sort (40) - : +- Exchange (39) - : +- Filter (38) - : +- Scan parquet (37) - +- Sort (45) - +- Exchange (44) - +- Project (43) - +- Filter (42) - +- Scan parquet (41) + Sort (50) + +- Exchange (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- Project (45) + +- SortMergeJoin Inner (44) + :- Sort (38) + : +- Exchange (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Sort (43) + +- Exchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -80,200 +78,192 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] (8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) SortExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(18) SortExecTransformer -Input [2]: [l_orderkey#X, l_shipmode#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(21) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(24) ColumnarExchange +(22) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(26) InputAdapter +(24) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(27) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(28) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(29) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(30) ColumnarExchange +(28) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(32) InputAdapter +(30) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(33) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(34) SortExecTransformer +(32) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(36) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(37) Scan parquet +(35) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(38) Filter +(36) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(39) Exchange +(37) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(38) Sort Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(41) Scan parquet +(39) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(42) Filter +(40) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(43) Project +(41) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(44) Exchange +(42) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(43) Sort Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(46) SortMergeJoin +(44) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(47) Project +(45) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(48) HashAggregate +(46) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(49) Exchange +(47) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(48) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(51) Exchange +(49) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(50) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(53) AdaptiveSparkPlan +(51) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt index 56efa184fae3..e7cde707d608 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt @@ -1,53 +1,51 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) - :- ^ SortExecTransformer (8) - : +- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) + :- ^ InputIteratorTransformer (7) + : +- ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- HashAggregate (49) - +- Project (48) - +- SortMergeJoin LeftOuter (47) - :- Sort (41) - : +- Exchange (40) - : +- Scan parquet (39) - +- Sort (46) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (53) + +- Exchange (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftOuter (45) + :- Sort (39) + : +- Exchange (38) + : +- Scan parquet (37) + +- Sort (44) + +- Exchange (43) + +- Project (42) + +- Filter (41) + +- Scan parquet (40) (1) Scan parquet @@ -78,227 +76,219 @@ Input [1]: [c_custkey#X] (7) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) SortExecTransformer -Input [1]: [c_custkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(9) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(10) NoopFilter +(9) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) ProjectExecTransformer +(10) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(12) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(12) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(13) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(14) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(15) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) SortExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(18) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(19) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(20) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(21) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(23) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(24) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(25) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(26) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(28) InputAdapter +(26) InputAdapter Input [2]: [c_count#X, count#X] -(29) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(30) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(31) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(32) ColumnarExchange +(30) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(31) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(34) InputAdapter +(32) InputAdapter Input [2]: [c_count#X, custdist#X] -(35) InputIteratorTransformer +(33) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(36) SortExecTransformer +(34) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(38) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(39) Scan parquet +(37) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(40) Exchange +(38) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(39) Sort Input [1]: [c_custkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(42) Scan parquet +(40) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(43) Filter +(41) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(44) Project +(42) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(45) Exchange +(43) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(44) Sort Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(47) SortMergeJoin +(45) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(48) Project +(46) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(49) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(50) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(51) HashAggregate +(49) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(52) Exchange +(50) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(51) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(54) Exchange +(52) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(53) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(56) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt index 4c247e24aee4..57965d52e44b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt @@ -1,40 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (39) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (25) - +- ^ ProjectExecTransformer (23) - +- ^ RegularHashAggregateExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (38) - +- HashAggregate (37) - +- Project (36) - +- SortMergeJoin Inner (35) - :- Sort (30) - : +- Exchange (29) - : +- Project (28) - : +- Filter (27) - : +- Scan parquet (26) - +- Sort (34) - +- Exchange (33) - +- Filter (32) - +- Scan parquet (31) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- SortMergeJoin Inner (33) + :- Sort (28) + : +- Exchange (27) + : +- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- Sort (32) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -70,148 +68,140 @@ Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] (8) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) SortExecTransformer -Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X, p_type#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(18) SortExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(21) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(22) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(23) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(24) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(25) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(26) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(27) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(28) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(29) Exchange +(27) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) Sort +(28) Sort Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(31) Scan parquet +(29) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(32) Filter +(30) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(33) Exchange +(31) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Sort +(32) Sort Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(35) SortMergeJoin +(33) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(36) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(37) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(38) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(39) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt index f2568bbdd3da..ee0566caef12 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt @@ -1,47 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- AQEShuffleRead (28) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (23) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (22) - +- ^ FilterExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- Project (43) - +- SortMergeJoin Inner (42) - :- Sort (33) - : +- Exchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Sort (41) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (43) + +- Exchange (42) + +- Project (41) + +- SortMergeJoin Inner (40) + :- Sort (31) + : +- Exchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Sort (39) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -77,332 +75,324 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] (8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) SortExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(13) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) ProjectExecTransformer +(13) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(15) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(16) ColumnarExchange +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(18) InputAdapter +(17) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(21) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(22) SortExecTransformer -Input [2]: [supplier_no#X, total_revenue#X] -Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 - -(23) ShuffledHashJoinExecTransformer +(21) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(24) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(25) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(26) ColumnarExchange +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(28) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(29) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) Exchange +(30) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) Sort +(31) Sort Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(34) Scan parquet +(32) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(33) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(34) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(36) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(40) Filter +(38) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) Sort +(39) Sort Input [2]: [supplier_no#X, total_revenue#X] Arguments: [supplier_no#X ASC NULLS FIRST], false, 0 -(42) SortMergeJoin +(40) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(43) Project +(41) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(44) Exchange +(42) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(43) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (71) +Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (62) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ RegularHashAggregateExecTransformer (59) - +- ^ ProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ NoopFilter (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (48) + +- ^ ProjectExecTransformer (47) + +- ^ NoopFilter (46) + +- ^ Scan parquet (45) +- == Initial Plan == - HashAggregate (70) - +- HashAggregate (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- Filter (64) - +- Scan parquet (63) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(47) Scan parquet +(45) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(48) NoopFilter +(46) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(49) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(50) FlushableHashAggregateExecTransformer +(48) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(58) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(59) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(60) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(61) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(62) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(63) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(64) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(65) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(66) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(67) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(69) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(70) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(71) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt index 16267e5c8350..b923af1d4c9c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt @@ -1,63 +1,61 @@ == Physical Plan == -AdaptiveSparkPlan (68) +AdaptiveSparkPlan (66) +- == Final Plan == - VeloxColumnarToRowExec (44) - +- ^ SortExecTransformer (42) - +- ^ InputIteratorTransformer (41) - +- ShuffleQueryStage (39), Statistics(X) - +- ColumnarExchange (38) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ ProjectExecTransformer (30) - +- ^ FlushableHashAggregateExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (42) + +- ^ SortExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- ^ ProjectExecTransformer (28) + +- ^ FlushableHashAggregateExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (67) - +- Exchange (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- HashAggregate (62) - +- Exchange (61) - +- HashAggregate (60) - +- Project (59) - +- SortMergeJoin Inner (58) - :- Sort (53) - : +- Exchange (52) - : +- BroadcastHashJoin LeftAnti BuildRight (51) - : :- Filter (46) - : : +- Scan parquet (45) - : +- BroadcastExchange (50) - : +- Project (49) - : +- Filter (48) - : +- Scan parquet (47) - +- Sort (57) - +- Exchange (56) - +- Filter (55) - +- Scan parquet (54) + Sort (65) + +- Exchange (64) + +- HashAggregate (63) + +- Exchange (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Exchange (59) + +- HashAggregate (58) + +- Project (57) + +- SortMergeJoin Inner (56) + :- Sort (51) + : +- Exchange (50) + : +- BroadcastHashJoin LeftAnti BuildRight (49) + : :- Filter (44) + : : +- Scan parquet (43) + : +- BroadcastExchange (48) + : +- Project (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Sort (55) + +- Exchange (54) + +- Filter (53) + +- Scan parquet (52) (1) Scan parquet @@ -93,275 +91,267 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] (8) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) SortExecTransformer -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(18) SortExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(21) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(24) ColumnarExchange +(22) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(23) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(26) InputAdapter +(24) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) InputIteratorTransformer +(25) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(28) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(29) FlushableHashAggregateExecTransformer +(27) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(30) ProjectExecTransformer +(28) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(31) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(32) ColumnarExchange +(30) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(34) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(35) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(36) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(37) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(38) ColumnarExchange +(36) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(37) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(40) InputAdapter +(38) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(41) InputIteratorTransformer +(39) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(42) SortExecTransformer +(40) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(43) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(44) VeloxColumnarToRowExec +(42) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(45) Scan parquet +(43) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(46) Filter +(44) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(47) Scan parquet +(45) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(48) Filter +(46) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(49) Project +(47) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(50) BroadcastExchange +(48) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(51) BroadcastHashJoin +(49) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(52) Exchange +(50) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) Sort +(51) Sort Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 -(54) Scan parquet +(52) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(55) Filter +(53) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(56) Exchange +(54) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(55) Sort Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(58) SortMergeJoin +(56) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(59) Project +(57) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(60) HashAggregate +(58) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(61) Exchange +(59) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(60) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(63) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(64) Exchange +(62) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(63) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(66) Exchange +(64) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(67) Sort +(65) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(68) AdaptiveSparkPlan +(66) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt index 7ca7ddb6ef7e..78f3ee40818e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt @@ -1,62 +1,59 @@ == Physical Plan == -AdaptiveSparkPlan (63) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ ProjectExecTransformer (38) - +- ^ RegularHashAggregateExecTransformer (37) - +- ^ RegularHashAggregateExecTransformer (36) - +- ^ ProjectExecTransformer (35) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (34) - :- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ NoopFilter (11) - : +- ^ Scan parquet (10) - +- ^ SortExecTransformer (33) - +- ^ FilterExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ NoopFilter (22) - +- ^ Scan parquet (21) + VeloxColumnarToRowExec (37) + +- ^ ProjectExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ RegularHashAggregateExecTransformer (33) + +- ^ ProjectExecTransformer (32) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) + :- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ FilterExecTransformer (30) + +- ^ ProjectExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ NoopFilter (20) + +- ^ Scan parquet (19) +- == Initial Plan == - HashAggregate (62) - +- HashAggregate (61) - +- Project (60) - +- SortMergeJoin Inner (59) - :- Project (51) - : +- SortMergeJoin Inner (50) - : :- Sort (44) - : : +- Exchange (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Sort (49) - : +- Exchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Sort (58) - +- Filter (57) - +- HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Filter (53) - +- Scan parquet (52) + HashAggregate (59) + +- HashAggregate (58) + +- Project (57) + +- SortMergeJoin Inner (56) + :- Project (48) + : +- SortMergeJoin Inner (47) + : :- Sort (41) + : : +- Exchange (40) + : : +- Filter (39) + : : +- Scan parquet (38) + : +- Sort (46) + : +- Exchange (45) + : +- Project (44) + : +- Filter (43) + : +- Scan parquet (42) + +- Sort (55) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -92,264 +89,252 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] (8) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) SortExecTransformer -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_container#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [1]: [p_partkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [p_partkey#X] -(18) SortExecTransformer -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(21) Scan parquet +(19) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(22) NoopFilter +(20) NoopFilter Input [2]: [l_partkey#X, l_quantity#X] Arguments: [l_partkey#X, l_quantity#X] -(23) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(24) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(25) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(26) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(28) InputAdapter +(26) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(29) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(30) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(31) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(32) FilterExecTransformer +(30) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(33) SortExecTransformer -Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(34) ShuffledHashJoinExecTransformer +(31) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(35) ProjectExecTransformer +(32) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(36) RegularHashAggregateExecTransformer +(33) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(37) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(38) ProjectExecTransformer +(35) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(39) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(40) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(41) Scan parquet +(38) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(42) Filter +(39) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(43) Exchange +(40) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(41) Sort Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(45) Scan parquet +(42) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(46) Filter +(43) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(47) Project +(44) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(48) Exchange +(45) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(46) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(50) SortMergeJoin +(47) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(51) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(52) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(53) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(54) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(55) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(57) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(58) Sort +(55) Sort Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(59) SortMergeJoin +(56) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(60) Project +(57) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(61) HashAggregate +(58) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(62) HashAggregate +(59) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(63) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt index 9a312ae38994..15aa5bdfe4b1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt @@ -1,103 +1,96 @@ == Physical Plan == -AdaptiveSparkPlan (111) +AdaptiveSparkPlan (104) +- == Final Plan == - VeloxColumnarToRowExec (71) - +- TakeOrderedAndProjectExecTransformer (70) - +- ^ RegularHashAggregateExecTransformer (68) - +- ^ RegularHashAggregateExecTransformer (67) - +- ^ ProjectExecTransformer (66) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (65) - :- ^ SortExecTransformer (46) - : +- ^ InputIteratorTransformer (45) - : +- ShuffleQueryStage (43), Statistics(X) - : +- ColumnarExchange (42) - : +- ^ ProjectExecTransformer (40) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (39) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (38) - : +- ^ InputIteratorTransformer (37) - : +- ShuffleQueryStage (35), Statistics(X) - : +- ColumnarExchange (34) - : +- ^ ProjectExecTransformer (32) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (31) - : :- ^ SortExecTransformer (18) - : : +- ^ InputIteratorTransformer (17) - : : +- ShuffleQueryStage (15), Statistics(X) - : : +- ColumnarExchange (14) - : : +- ^ ProjectExecTransformer (12) - : : +- ^ NoopFilter (11) - : : +- ^ Scan parquet (10) - : +- ^ SortExecTransformer (30) - : +- ^ ProjectExecTransformer (29) - : +- ^ FilterExecTransformer (28) - : +- ^ RegularHashAggregateExecTransformer (27) - : +- ^ InputIteratorTransformer (26) - : +- ShuffleQueryStage (24), Statistics(X) - : +- ColumnarExchange (23) - : +- ^ ProjectExecTransformer (21) - : +- ^ FlushableHashAggregateExecTransformer (20) - : +- ^ Scan parquet (19) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (64) - :- ^ SortExecTransformer (55) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ SortExecTransformer (63) - +- ^ ProjectExecTransformer (62) - +- ^ FilterExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ InputIteratorTransformer (59) - +- ShuffleQueryStage (57), Statistics(X) - +- ReusedExchange (56) + VeloxColumnarToRowExec (64) + +- TakeOrderedAndProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) + :- ^ InputIteratorTransformer (41) + : +- ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : :- ^ InputIteratorTransformer (16) + : : +- ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) + :- ^ InputIteratorTransformer (49) + : +- ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (110) - +- HashAggregate (109) - +- HashAggregate (108) - +- Project (107) - +- SortMergeJoin Inner (106) - :- Sort (93) - : +- Exchange (92) - : +- Project (91) - : +- SortMergeJoin Inner (90) - : :- Sort (75) - : : +- Exchange (74) - : : +- Filter (73) - : : +- Scan parquet (72) - : +- Sort (89) - : +- Exchange (88) - : +- SortMergeJoin LeftSemi (87) - : :- Sort (79) - : : +- Exchange (78) - : : +- Filter (77) - : : +- Scan parquet (76) - : +- Sort (86) - : +- Project (85) - : +- Filter (84) - : +- HashAggregate (83) - : +- Exchange (82) - : +- HashAggregate (81) - : +- Scan parquet (80) - +- SortMergeJoin LeftSemi (105) - :- Sort (97) - : +- Exchange (96) - : +- Filter (95) - : +- Scan parquet (94) - +- Sort (104) - +- Project (103) - +- Filter (102) - +- HashAggregate (101) - +- Exchange (100) - +- HashAggregate (99) - +- Scan parquet (98) + TakeOrderedAndProject (103) + +- HashAggregate (102) + +- HashAggregate (101) + +- Project (100) + +- SortMergeJoin Inner (99) + :- Sort (86) + : +- Exchange (85) + : +- Project (84) + : +- SortMergeJoin Inner (83) + : :- Sort (68) + : : +- Exchange (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Sort (82) + : +- Exchange (81) + : +- SortMergeJoin LeftSemi (80) + : :- Sort (72) + : : +- Exchange (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- Sort (79) + : +- Project (78) + : +- Filter (77) + : +- HashAggregate (76) + : +- Exchange (75) + : +- HashAggregate (74) + : +- Scan parquet (73) + +- SortMergeJoin LeftSemi (98) + :- Sort (90) + : +- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Sort (97) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -133,470 +126,442 @@ Input [2]: [c_custkey#X, c_name#X] (8) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) SortExecTransformer -Input [2]: [c_custkey#X, c_name#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(16) InputAdapter -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] - -(17) InputIteratorTransformer +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(18) SortExecTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(19) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(20) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(28) FilterExecTransformer +(26) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(29) ProjectExecTransformer +(27) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(30) SortExecTransformer -Input [1]: [l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(31) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(32) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(33) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(34) ColumnarExchange +(31) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(35) ShuffleQueryStage +(32) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(36) InputAdapter +(33) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) InputIteratorTransformer +(34) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(38) SortExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(39) ShuffledHashJoinExecTransformer +(35) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(40) ProjectExecTransformer +(36) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(41) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(42) ColumnarExchange +(38) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(43) ShuffleQueryStage +(39) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(44) InputAdapter +(40) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(45) InputIteratorTransformer +(41) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(46) SortExecTransformer -Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(47) Scan parquet +(42) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(48) NoopFilter +(43) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(49) ProjectExecTransformer +(44) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(50) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(51) ColumnarExchange +(46) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(47) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(53) InputAdapter +(48) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(54) InputIteratorTransformer +(49) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(55) SortExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(56) ReusedExchange [Reuses operator id: 23] +(50) ReusedExchange [Reuses operator id: 21] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(58) InputAdapter +(52) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(59) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(60) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(61) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(62) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(63) SortExecTransformer -Input [1]: [l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(64) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(65) ShuffledHashJoinExecTransformer +(58) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(66) ProjectExecTransformer +(59) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(67) RegularHashAggregateExecTransformer +(60) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(68) RegularHashAggregateExecTransformer +(61) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(69) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(70) TakeOrderedAndProjectExecTransformer +(63) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(71) VeloxColumnarToRowExec +(64) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(72) Scan parquet +(65) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(73) Filter +(66) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(74) Exchange +(67) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) Sort +(68) Sort Input [2]: [c_custkey#X, c_name#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(76) Scan parquet +(69) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(77) Filter +(70) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(78) Exchange +(71) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Sort +(72) Sort Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(80) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(81) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(82) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(83) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(84) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(85) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(86) Sort +(79) Sort Input [1]: [l_orderkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(87) SortMergeJoin +(80) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(88) Exchange +(81) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Sort +(82) Sort Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(90) SortMergeJoin +(83) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(91) Project +(84) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(92) Exchange +(85) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Sort +(86) Sort Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(94) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(96) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(90) Sort Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(98) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(99) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(100) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(102) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(103) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(104) Sort +(97) Sort Input [1]: [l_orderkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(105) SortMergeJoin +(98) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(106) SortMergeJoin +(99) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(107) Project +(100) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(108) HashAggregate +(101) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(109) HashAggregate +(102) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(110) TakeOrderedAndProject +(103) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(111) AdaptiveSparkPlan +(104) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt index 7234e3ae5053..17817a746612 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt @@ -1,39 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (24) - +- ^ RegularHashAggregateExecTransformer (22) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - HashAggregate (37) - +- HashAggregate (36) - +- Project (35) - +- SortMergeJoin Inner (34) - :- Sort (29) - : +- Exchange (28) - : +- Project (27) - : +- Filter (26) - : +- Scan parquet (25) - +- Sort (33) - +- Exchange (32) - +- Filter (31) - +- Scan parquet (30) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- SortMergeJoin Inner (32) + :- Sort (27) + : +- Exchange (26) + : +- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- Sort (31) + +- Exchange (30) + +- Filter (29) + +- Scan parquet (28) (1) Scan parquet @@ -69,144 +67,136 @@ Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] (8) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) SortExecTransformer -Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(18) SortExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(21) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(22) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(24) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(25) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(26) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(27) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(28) Exchange +(26) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(29) Sort +(27) Sort Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(30) Scan parquet +(28) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(31) Filter +(29) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(32) Exchange +(30) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(33) Sort +(31) Sort Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(34) SortMergeJoin +(32) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(35) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(36) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(37) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(38) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt index f9c1a31a3d75..a8f03b327dbe 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt @@ -1,129 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (143) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (93) - +- AQEShuffleRead (92) - +- ShuffleQueryStage (91), Statistics(X) - +- ColumnarExchange (90) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ SortExecTransformer (77) - : +- ^ InputIteratorTransformer (76) - : +- ShuffleQueryStage (74), Statistics(X) - : +- ColumnarExchange (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (70) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ SortExecTransformer (35) - : : +- ^ InputIteratorTransformer (34) - : : +- ShuffleQueryStage (32), Statistics(X) - : : +- ColumnarExchange (31) - : : +- ^ ProjectExecTransformer (29) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : : :- ^ SortExecTransformer (18) - : : : +- ^ InputIteratorTransformer (17) - : : : +- ShuffleQueryStage (15), Statistics(X) - : : : +- ColumnarExchange (14) - : : : +- ^ ProjectExecTransformer (12) - : : : +- ^ NoopFilter (11) - : : : +- ^ Scan parquet (10) - : : +- ^ SortExecTransformer (27) - : : +- ^ InputIteratorTransformer (26) - : : +- ShuffleQueryStage (24), Statistics(X) - : : +- ColumnarExchange (23) - : : +- ^ ProjectExecTransformer (21) - : : +- ^ NoopFilter (20) - : : +- ^ Scan parquet (19) - : +- ^ SortExecTransformer (61) - : +- ^ InputIteratorTransformer (60) - : +- ShuffleQueryStage (58), Statistics(X) - : +- ColumnarExchange (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ FilterExecTransformer (54) - : +- ^ ProjectExecTransformer (53) - : +- ^ RegularHashAggregateExecTransformer (52) - : +- ^ RegularHashAggregateExecTransformer (51) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (50) - : :- ^ SortExecTransformer (44) - : : +- ^ InputIteratorTransformer (43) - : : +- ShuffleQueryStage (41), Statistics(X) - : : +- ColumnarExchange (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ NoopFilter (37) - : : +- ^ Scan parquet (36) - : +- ^ SortExecTransformer (49) - : +- ^ InputIteratorTransformer (48) - : +- ShuffleQueryStage (46), Statistics(X) - : +- ReusedExchange (45) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (83) + +- AQEShuffleRead (82) + +- ShuffleQueryStage (81), Statistics(X) + +- ColumnarExchange (80) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) + : :- ^ InputIteratorTransformer (31) + : : +- ShuffleQueryStage (29), Statistics(X) + : : +- ColumnarExchange (28) + : : +- ^ ProjectExecTransformer (26) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) + : : :- ^ InputIteratorTransformer (16) + : : : +- ShuffleQueryStage (14), Statistics(X) + : : : +- ColumnarExchange (13) + : : : +- ^ ProjectExecTransformer (11) + : : : +- ^ NoopFilter (10) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (24) + : : +- ShuffleQueryStage (22), Statistics(X) + : : +- ColumnarExchange (21) + : : +- ^ ProjectExecTransformer (19) + : : +- ^ NoopFilter (18) + : : +- ^ Scan parquet (17) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ ProjectExecTransformer (47) + : +- ^ RegularHashAggregateExecTransformer (46) + : +- ^ RegularHashAggregateExecTransformer (45) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ NoopFilter (33) + : : +- ^ Scan parquet (32) + : +- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ReusedExchange (40) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (142) - +- Exchange (141) - +- Project (140) - +- SortMergeJoin Inner (139) - :- Sort (133) - : +- Exchange (132) - : +- Project (131) - : +- SortMergeJoin LeftSemi (130) - : :- Sort (97) - : : +- Exchange (96) - : : +- Filter (95) - : : +- Scan parquet (94) - : +- Sort (129) - : +- Exchange (128) - : +- Project (127) - : +- SortMergeJoin Inner (126) - : :- Sort (109) - : : +- Exchange (108) - : : +- SortMergeJoin LeftSemi (107) - : : :- Sort (101) - : : : +- Exchange (100) - : : : +- Filter (99) - : : : +- Scan parquet (98) - : : +- Sort (106) - : : +- Exchange (105) - : : +- Project (104) - : : +- Filter (103) - : : +- Scan parquet (102) - : +- Sort (125) - : +- Exchange (124) - : +- Filter (123) - : +- HashAggregate (122) - : +- HashAggregate (121) - : +- SortMergeJoin LeftSemi (120) - : :- Sort (114) - : : +- Exchange (113) - : : +- Project (112) - : : +- Filter (111) - : : +- Scan parquet (110) - : +- Sort (119) - : +- Exchange (118) - : +- Project (117) - : +- Filter (116) - : +- Scan parquet (115) - +- Sort (138) - +- Exchange (137) - +- Project (136) - +- Filter (135) - +- Scan parquet (134) + Sort (132) + +- Exchange (131) + +- Project (130) + +- SortMergeJoin Inner (129) + :- Sort (123) + : +- Exchange (122) + : +- Project (121) + : +- SortMergeJoin LeftSemi (120) + : :- Sort (87) + : : +- Exchange (86) + : : +- Filter (85) + : : +- Scan parquet (84) + : +- Sort (119) + : +- Exchange (118) + : +- Project (117) + : +- SortMergeJoin Inner (116) + : :- Sort (99) + : : +- Exchange (98) + : : +- SortMergeJoin LeftSemi (97) + : : :- Sort (91) + : : : +- Exchange (90) + : : : +- Filter (89) + : : : +- Scan parquet (88) + : : +- Sort (96) + : : +- Exchange (95) + : : +- Project (94) + : : +- Filter (93) + : : +- Scan parquet (92) + : +- Sort (115) + : +- Exchange (114) + : +- Filter (113) + : +- HashAggregate (112) + : +- HashAggregate (111) + : +- SortMergeJoin LeftSemi (110) + : :- Sort (104) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Sort (109) + : +- Exchange (108) + : +- Project (107) + : +- Filter (106) + : +- Scan parquet (105) + +- Sort (128) + +- Exchange (127) + +- Project (126) + +- Filter (125) + +- Scan parquet (124) (1) Scan parquet @@ -159,584 +149,544 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] (8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) SortExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(16) InputAdapter -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] - -(17) InputIteratorTransformer +(15) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(18) SortExecTransformer +(16) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 -(19) Scan parquet +(17) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(20) NoopFilter +(18) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [1]: [p_partkey#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [1]: [p_partkey#X] -(27) SortExecTransformer -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(28) ShuffledHashJoinExecTransformer +(25) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(35) SortExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 - -(36) Scan parquet +(32) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(37) NoopFilter +(33) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(38) ProjectExecTransformer +(34) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(39) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(40) ColumnarExchange +(36) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(41) ShuffleQueryStage +(37) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(42) InputAdapter +(38) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(43) InputIteratorTransformer +(39) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(44) SortExecTransformer -Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(45) ReusedExchange [Reuses operator id: 23] +(40) ReusedExchange [Reuses operator id: 21] Output [1]: [p_partkey#X] -(46) ShuffleQueryStage +(41) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(47) InputAdapter -Input [1]: [p_partkey#X] - -(48) InputIteratorTransformer +(42) InputAdapter Input [1]: [p_partkey#X] -(49) SortExecTransformer +(43) InputIteratorTransformer Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(50) ShuffledHashJoinExecTransformer +(44) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(51) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(53) ProjectExecTransformer +(47) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(54) FilterExecTransformer +(48) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(55) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(56) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(57) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(59) InputAdapter +(53) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(60) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(61) SortExecTransformer -Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 - -(62) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(67) InputAdapter +(60) InputAdapter Input [1]: [ps_suppkey#X] -(68) InputIteratorTransformer -Input [1]: [ps_suppkey#X] - -(69) SortExecTransformer +(61) InputIteratorTransformer Input [1]: [ps_suppkey#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(75) InputAdapter +(67) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(76) InputIteratorTransformer +(68) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(77) SortExecTransformer -Input [3]: [s_name#X, s_address#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(78) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(79) NoopFilter +(70) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(81) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(82) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(84) InputAdapter +(75) InputAdapter Input [1]: [n_nationkey#X] -(85) InputIteratorTransformer +(76) InputIteratorTransformer Input [1]: [n_nationkey#X] -(86) SortExecTransformer -Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 - -(87) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) ProjectExecTransformer +(78) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(89) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(90) ColumnarExchange +(80) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(91) ShuffleQueryStage +(81) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(92) AQEShuffleRead +(82) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(93) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(94) Scan parquet +(84) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(95) Filter +(85) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(96) Exchange +(86) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(87) Sort Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(98) Scan parquet +(88) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(99) Filter +(89) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(100) Exchange +(90) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Sort +(91) Sort Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X ASC NULLS FIRST], false, 0 -(102) Scan parquet +(92) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(103) Filter +(93) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(104) Project +(94) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(105) Exchange +(95) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Sort +(96) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(107) SortMergeJoin +(97) SortMergeJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(108) Exchange +(98) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Sort +(99) Sort Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X ASC NULLS FIRST, ps_suppkey#X ASC NULLS FIRST], false, 0 -(110) Scan parquet +(100) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(111) Filter +(101) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(112) Project +(102) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(113) Exchange +(103) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) Sort +(104) Sort Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(115) Scan parquet +(105) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(116) Filter +(106) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(117) Project +(107) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(118) Exchange +(108) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) Sort +(109) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(120) SortMergeJoin +(110) SortMergeJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(121) HashAggregate +(111) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(122) HashAggregate +(112) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(123) Filter +(113) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(124) Exchange +(114) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(115) Sort Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: [l_partkey#X ASC NULLS FIRST, l_suppkey#X ASC NULLS FIRST], false, 0 -(126) SortMergeJoin +(116) SortMergeJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(127) Project +(117) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(128) Exchange +(118) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(119) Sort Input [1]: [ps_suppkey#X] Arguments: [ps_suppkey#X ASC NULLS FIRST], false, 0 -(130) SortMergeJoin +(120) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(131) Project +(121) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(132) Exchange +(122) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(123) Sort Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(134) Scan parquet +(124) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(135) Filter +(125) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(136) Project +(126) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(137) Exchange +(127) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(138) Sort +(128) Sort Input [1]: [n_nationkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(139) SortMergeJoin +(129) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(140) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(141) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(143) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt index 49f8b39fe06a..652f723413f7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt @@ -1,121 +1,112 @@ == Physical Plan == -AdaptiveSparkPlan (136) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ RegularHashAggregateExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ ProjectExecTransformer (82) - +- ^ FlushableHashAggregateExecTransformer (81) - +- ^ ProjectExecTransformer (80) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) - :- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ SortExecTransformer (52) - : : +- ^ InputIteratorTransformer (51) - : : +- ShuffleQueryStage (49), Statistics(X) - : : +- ColumnarExchange (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) - : : :- ^ SortExecTransformer (9) - : : : +- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ SortExecTransformer (44) - : : +- ^ InputIteratorTransformer (43) - : : +- ShuffleQueryStage (41), Statistics(X) - : : +- ColumnarExchange (40) - : : +- ^ ProjectExecTransformer (38) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) - : : : :- ^ SortExecTransformer (18) - : : : : +- ^ InputIteratorTransformer (17) - : : : : +- ShuffleQueryStage (15), Statistics(X) - : : : : +- ColumnarExchange (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ SortExecTransformer (26) - : : : +- ^ InputIteratorTransformer (25) - : : : +- ShuffleQueryStage (23), Statistics(X) - : : : +- ColumnarExchange (22) - : : : +- ^ ProjectExecTransformer (20) - : : : +- ^ Scan parquet (19) - : : +- ^ SortExecTransformer (36) - : : +- ^ InputIteratorTransformer (35) - : : +- ShuffleQueryStage (33), Statistics(X) - : : +- ColumnarExchange (32) - : : +- ^ ProjectExecTransformer (30) - : : +- ^ NoopFilter (29) - : : +- ^ Scan parquet (28) - : +- ^ SortExecTransformer (61) - : +- ^ InputIteratorTransformer (60) - : +- ShuffleQueryStage (58), Statistics(X) - : +- ColumnarExchange (57) - : +- ^ ProjectExecTransformer (55) - : +- ^ NoopFilter (54) - : +- ^ Scan parquet (53) - +- ^ SortExecTransformer (78) - +- ^ InputIteratorTransformer (77) - +- ShuffleQueryStage (75), Statistics(X) - +- ColumnarExchange (74) - +- ^ ProjectExecTransformer (72) - +- ^ NoopFilter (71) - +- ^ Scan parquet (70) + VeloxColumnarToRowExec (81) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + :- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) + : :- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ShuffleQueryStage (37), Statistics(X) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ShuffleQueryStage (21), Statistics(X) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ShuffleQueryStage (30), Statistics(X) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == - TakeOrderedAndProject (135) - +- HashAggregate (134) - +- Exchange (133) - +- HashAggregate (132) - +- Project (131) - +- SortMergeJoin Inner (130) - :- Sort (124) - : +- Exchange (123) - : +- Project (122) - : +- SortMergeJoin Inner (121) - : :- Sort (115) - : : +- Exchange (114) - : : +- Project (113) - : : +- SortMergeJoin Inner (112) - : : :- Sort (94) - : : : +- Exchange (93) - : : : +- Filter (92) - : : : +- Scan parquet (91) - : : +- Sort (111) - : : +- Exchange (110) - : : +- SortMergeJoin LeftAnti (109) - : : :- SortMergeJoin LeftSemi (103) - : : : :- Sort (99) - : : : : +- Exchange (98) - : : : : +- Project (97) - : : : : +- Filter (96) - : : : : +- Scan parquet (95) - : : : +- Sort (102) - : : : +- Exchange (101) - : : : +- Scan parquet (100) - : : +- Sort (108) - : : +- Exchange (107) - : : +- Project (106) - : : +- Filter (105) - : : +- Scan parquet (104) - : +- Sort (120) - : +- Exchange (119) - : +- Project (118) - : +- Filter (117) - : +- Scan parquet (116) - +- Sort (129) - +- Exchange (128) - +- Project (127) - +- Filter (126) - +- Scan parquet (125) + TakeOrderedAndProject (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- SortMergeJoin Inner (121) + :- Sort (115) + : +- Exchange (114) + : +- Project (113) + : +- SortMergeJoin Inner (112) + : :- Sort (106) + : : +- Exchange (105) + : : +- Project (104) + : : +- SortMergeJoin Inner (103) + : : :- Sort (85) + : : : +- Exchange (84) + : : : +- Filter (83) + : : : +- Scan parquet (82) + : : +- Sort (102) + : : +- Exchange (101) + : : +- SortMergeJoin LeftAnti (100) + : : :- SortMergeJoin LeftSemi (94) + : : : :- Sort (90) + : : : : +- Exchange (89) + : : : : +- Project (88) + : : : : +- Filter (87) + : : : : +- Scan parquet (86) + : : : +- Sort (93) + : : : +- Exchange (92) + : : : +- Scan parquet (91) + : : +- Sort (99) + : : +- Exchange (98) + : : +- Project (97) + : : +- Filter (96) + : : +- Scan parquet (95) + : +- Sort (111) + : +- Exchange (110) + : +- Project (109) + : +- Filter (108) + : +- Scan parquet (107) + +- Sort (120) + +- Exchange (119) + +- Project (118) + +- Filter (117) + +- Scan parquet (116) (1) Scan parquet @@ -151,558 +142,522 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] (8) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) SortExecTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(18) SortExecTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(19) Scan parquet +(17) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(24) InputAdapter -Input [2]: [l_orderkey#X, l_suppkey#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(27) ShuffledHashJoinExecTransformer +(24) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(28) Scan parquet +(25) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(29) NoopFilter +(26) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(30) ProjectExecTransformer +(27) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(31) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(32) ColumnarExchange +(29) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(34) InputAdapter +(31) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(35) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(36) SortExecTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(37) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(38) ProjectExecTransformer +(34) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(40) ColumnarExchange +(36) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(41) ShuffleQueryStage +(37) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(42) InputAdapter +(38) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(43) InputIteratorTransformer +(39) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(44) SortExecTransformer -Input [2]: [l_orderkey#X, l_suppkey#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 - -(45) ShuffledHashJoinExecTransformer +(40) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(50) InputAdapter -Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] - -(51) InputIteratorTransformer +(45) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(52) SortExecTransformer +(46) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(53) Scan parquet +(47) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(54) NoopFilter +(48) NoopFilter Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: [o_orderkey#X, o_orderstatus#X] -(55) ProjectExecTransformer +(49) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(56) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(57) ColumnarExchange +(51) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(52) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(59) InputAdapter +(53) InputAdapter Input [1]: [o_orderkey#X] -(60) InputIteratorTransformer +(54) InputIteratorTransformer Input [1]: [o_orderkey#X] -(61) SortExecTransformer -Input [1]: [o_orderkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(62) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(60) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(68) InputIteratorTransformer +(61) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(69) SortExecTransformer -Input [2]: [s_name#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(70) Scan parquet +(62) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(71) NoopFilter +(63) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(72) ProjectExecTransformer +(64) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(73) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(74) ColumnarExchange +(66) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(75) ShuffleQueryStage +(67) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(76) InputAdapter -Input [1]: [n_nationkey#X] - -(77) InputIteratorTransformer +(68) InputAdapter Input [1]: [n_nationkey#X] -(78) SortExecTransformer +(69) InputIteratorTransformer Input [1]: [n_nationkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(79) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(81) FlushableHashAggregateExecTransformer +(72) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(82) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(83) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(84) ColumnarExchange +(75) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(86) InputAdapter +(77) InputAdapter Input [2]: [s_name#X, count#X] -(87) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(88) RegularHashAggregateExecTransformer +(79) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(89) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(90) VeloxColumnarToRowExec +(81) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(91) Scan parquet +(82) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(83) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(84) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Sort +(85) Sort Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(95) Scan parquet +(86) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(96) Filter +(87) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(97) Project +(88) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(98) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(90) Sort Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(100) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(101) Exchange +(92) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Sort +(93) Sort Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(103) SortMergeJoin +(94) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(104) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(105) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(106) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(107) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Sort +(99) Sort Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(109) SortMergeJoin +(100) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(110) Exchange +(101) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Sort +(102) Sort Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(112) SortMergeJoin +(103) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(113) Project +(104) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(114) Exchange +(105) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Sort +(106) Sort Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(116) Scan parquet +(107) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(117) Filter +(108) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(118) Project +(109) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(119) Exchange +(110) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(111) Sort Input [1]: [o_orderkey#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(121) SortMergeJoin +(112) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(122) Project +(113) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(123) Exchange +(114) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) Sort +(115) Sort Input [2]: [s_name#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(125) Scan parquet +(116) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(126) Filter +(117) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(127) Project +(118) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(128) Exchange +(119) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(120) Sort Input [1]: [n_nationkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(130) SortMergeJoin +(121) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(131) Project +(122) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(132) HashAggregate +(123) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(133) Exchange +(124) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(134) HashAggregate +(125) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(135) TakeOrderedAndProject +(126) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(136) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt index 8c9d473d085d..41f480c0af15 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt @@ -1,47 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (48) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ ProjectExecTransformer (19) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (33) + +- ^ SortExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (15) + +- ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- SortMergeJoin LeftAnti (43) - :- Sort (39) - : +- Exchange (38) - : +- Filter (37) - : +- Scan parquet (36) - +- Sort (42) - +- Exchange (41) - +- Scan parquet (40) + Sort (47) + +- Exchange (46) + +- HashAggregate (45) + +- Exchange (44) + +- HashAggregate (43) + +- Project (42) + +- SortMergeJoin LeftAnti (41) + :- Sort (37) + : +- Exchange (36) + : +- Filter (35) + : +- Scan parquet (34) + +- Sort (40) + +- Exchange (39) + +- Scan parquet (38) (1) Scan parquet @@ -77,300 +75,292 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] (8) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) SortExecTransformer -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(12) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(12) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(13) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(15) InputAdapter +(14) InputAdapter Input [1]: [o_custkey#X] -(16) InputIteratorTransformer +(15) InputIteratorTransformer Input [1]: [o_custkey#X] -(17) SortExecTransformer -Input [1]: [o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(18) ShuffledHashJoinExecTransformer +(16) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(19) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(20) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(27) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(28) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(29) ColumnarExchange +(27) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(28) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(31) InputAdapter +(29) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(32) InputIteratorTransformer +(30) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(33) SortExecTransformer +(31) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(35) VeloxColumnarToRowExec +(33) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(36) Scan parquet +(34) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(37) Filter +(35) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(38) Exchange +(36) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Sort +(37) Sort Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(40) Scan parquet +(38) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(41) Exchange +(39) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) Sort +(40) Sort Input [1]: [o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(43) SortMergeJoin +(41) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(44) Project +(42) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(45) HashAggregate +(43) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(46) Exchange +(44) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(45) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(48) Exchange +(46) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(47) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(48) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (69) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (62) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ InputIteratorTransformer (59) - +- ShuffleQueryStage (57), Statistics(X) - +- ColumnarExchange (56) - +- ^ FlushableHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ NoopFilter (52) - +- ^ Scan parquet (51) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ InputIteratorTransformer (57) + +- ShuffleQueryStage (55), Statistics(X) + +- ColumnarExchange (54) + +- ^ FlushableHashAggregateExecTransformer (52) + +- ^ ProjectExecTransformer (51) + +- ^ NoopFilter (50) + +- ^ Scan parquet (49) +- == Initial Plan == - HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- Filter (64) - +- Scan parquet (63) + HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(51) Scan parquet +(49) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) NoopFilter +(50) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(53) ProjectExecTransformer +(51) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) FlushableHashAggregateExecTransformer +(52) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(56) ColumnarExchange +(54) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(55) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(58) InputAdapter +(56) InputAdapter Input [2]: [sum#X, count#X] -(59) InputIteratorTransformer +(57) InputIteratorTransformer Input [2]: [sum#X, count#X] -(60) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(61) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(62) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(63) Scan parquet +(61) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(64) Filter +(62) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(65) Project +(63) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(66) HashAggregate +(64) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(67) Exchange +(65) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(66) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(69) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt index 1d66056a3537..d3ab264e76c4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt @@ -1,64 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ RegularHashAggregateExecTransformer (38) - +- ^ ProjectExecTransformer (37) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - :- ^ SortExecTransformer (26) - : +- ^ InputIteratorTransformer (25) - : +- ShuffleQueryStage (23), Statistics(X) - : +- ColumnarExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : :- ^ SortExecTransformer (9) - : : +- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ SortExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ NoopFilter (11) - : +- ^ Scan parquet (10) - +- ^ SortExecTransformer (35) - +- ^ InputIteratorTransformer (34) - +- ShuffleQueryStage (32), Statistics(X) - +- ColumnarExchange (31) - +- ^ ProjectExecTransformer (29) - +- ^ NoopFilter (28) - +- ^ Scan parquet (27) + VeloxColumnarToRowExec (39) + +- TakeOrderedAndProjectExecTransformer (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + :- ^ InputIteratorTransformer (23) + : +- ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : :- ^ InputIteratorTransformer (8) + : : +- ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Project (63) - +- SortMergeJoin Inner (62) - :- Sort (56) - : +- Exchange (55) - : +- Project (54) - : +- SortMergeJoin Inner (53) - : :- Sort (48) - : : +- Exchange (47) - : : +- Project (46) - : : +- Filter (45) - : : +- Scan parquet (44) - : +- Sort (52) - : +- Exchange (51) - : +- Filter (50) - : +- Scan parquet (49) - +- Sort (61) - +- Exchange (60) - +- Project (59) - +- Filter (58) - +- Scan parquet (57) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- SortMergeJoin Inner (58) + :- Sort (52) + : +- Exchange (51) + : +- Project (50) + : +- SortMergeJoin Inner (49) + : :- Sort (44) + : : +- Exchange (43) + : : +- Project (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Sort (48) + : +- Exchange (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Sort (57) + +- Exchange (56) + +- Project (55) + +- Filter (54) + +- Scan parquet (53) (1) Scan parquet @@ -94,266 +90,250 @@ Input [1]: [c_custkey#X] (8) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) SortExecTransformer -Input [1]: [c_custkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(18) SortExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(26) SortExecTransformer -Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(27) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(33) InputAdapter -Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] - -(34) InputIteratorTransformer +(30) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(35) SortExecTransformer +(31) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(38) RegularHashAggregateExecTransformer +(34) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(40) ProjectExecTransformer +(36) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(41) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(38) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(43) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(44) Scan parquet +(40) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(45) Filter +(41) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(46) Project +(42) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(47) Exchange +(43) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(44) Sort Input [1]: [c_custkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(49) Scan parquet +(45) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(50) Filter +(46) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(51) Exchange +(47) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) Sort +(48) Sort Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(53) SortMergeJoin +(49) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(54) Project +(50) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(55) Exchange +(51) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Sort +(52) Sort Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(57) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(58) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(59) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(60) Exchange +(56) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) Sort +(57) Sort Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(62) SortMergeJoin +(58) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(63) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(64) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(65) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(66) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(67) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt index 83ba6833b3b5..6a1e5a585663 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt @@ -1,51 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) - :- ^ SortExecTransformer (9) - : +- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ SortExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ NoopFilter (11) - +- ^ Scan parquet (10) + VeloxColumnarToRowExec (34) + +- ^ SortExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (16) + +- ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (53) - +- Exchange (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- SortMergeJoin LeftSemi (47) - :- Sort (41) - : +- Exchange (40) - : +- Project (39) - : +- Filter (38) - : +- Scan parquet (37) - +- Sort (46) - +- Exchange (45) - +- Project (44) - +- Filter (43) - +- Scan parquet (42) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- SortMergeJoin LeftSemi (45) + :- Sort (39) + : +- Exchange (38) + : +- Project (37) + : +- Filter (36) + : +- Scan parquet (35) + +- Sort (44) + +- Exchange (43) + +- Project (42) + +- Filter (41) + +- Scan parquet (40) (1) Scan parquet @@ -81,204 +79,196 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] (8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) SortExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [1]: [l_orderkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [l_orderkey#X] -(18) SortExecTransformer -Input [1]: [l_orderkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(21) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(24) ColumnarExchange +(22) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(23) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(26) InputAdapter +(24) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(27) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(28) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(29) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(30) ColumnarExchange +(28) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(32) InputAdapter +(30) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(33) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(34) SortExecTransformer +(32) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(35) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(36) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(37) Scan parquet +(35) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(38) Filter +(36) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(39) Project +(37) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(40) Exchange +(38) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(39) Sort Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(42) Scan parquet +(40) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(43) Filter +(41) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(44) Project +(42) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(45) Exchange +(43) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) Sort +(44) Sort Input [1]: [l_orderkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(47) SortMergeJoin +(45) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(48) Project +(46) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(49) HashAggregate +(47) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(50) Exchange +(48) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(49) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(52) Exchange +(50) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) Sort +(51) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(54) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt index dfa1d8ebb98f..1b8a65c7d87d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt @@ -1,135 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (154) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (104) - +- ^ SortExecTransformer (102) - +- ^ InputIteratorTransformer (101) - +- ShuffleQueryStage (99), Statistics(X) - +- ColumnarExchange (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ SortExecTransformer (77) - : +- ^ InputIteratorTransformer (76) - : +- ShuffleQueryStage (74), Statistics(X) - : +- ColumnarExchange (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ SortExecTransformer (60) - : : +- ^ InputIteratorTransformer (59) - : : +- ShuffleQueryStage (57), Statistics(X) - : : +- ColumnarExchange (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ SortExecTransformer (43) - : : : +- ^ InputIteratorTransformer (42) - : : : +- ShuffleQueryStage (40), Statistics(X) - : : : +- ColumnarExchange (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) - : : : :- ^ SortExecTransformer (26) - : : : : +- ^ InputIteratorTransformer (25) - : : : : +- ShuffleQueryStage (23), Statistics(X) - : : : : +- ColumnarExchange (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ SortExecTransformer (9) - : : : : : +- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ SortExecTransformer (18) - : : : : +- ^ InputIteratorTransformer (17) - : : : : +- ShuffleQueryStage (15), Statistics(X) - : : : : +- ColumnarExchange (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ SortExecTransformer (35) - : : : +- ^ InputIteratorTransformer (34) - : : : +- ShuffleQueryStage (32), Statistics(X) - : : : +- ColumnarExchange (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ SortExecTransformer (52) - : : +- ^ InputIteratorTransformer (51) - : : +- ShuffleQueryStage (49), Statistics(X) - : : +- ColumnarExchange (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (153) - +- Exchange (152) - +- HashAggregate (151) - +- Exchange (150) - +- HashAggregate (149) - +- Project (148) - +- SortMergeJoin Inner (147) - :- Sort (141) - : +- Exchange (140) - : +- Project (139) - : +- SortMergeJoin Inner (138) - : :- Sort (133) - : : +- Exchange (132) - : : +- Project (131) - : : +- SortMergeJoin Inner (130) - : : :- Sort (125) - : : : +- Exchange (124) - : : : +- Project (123) - : : : +- SortMergeJoin Inner (122) - : : : :- Sort (117) - : : : : +- Exchange (116) - : : : : +- Project (115) - : : : : +- SortMergeJoin Inner (114) - : : : : :- Sort (108) - : : : : : +- Exchange (107) - : : : : : +- Filter (106) - : : : : : +- Scan parquet (105) - : : : : +- Sort (113) - : : : : +- Exchange (112) - : : : : +- Project (111) - : : : : +- Filter (110) - : : : : +- Scan parquet (109) - : : : +- Sort (121) - : : : +- Exchange (120) - : : : +- Filter (119) - : : : +- Scan parquet (118) - : : +- Sort (129) - : : +- Exchange (128) - : : +- Filter (127) - : : +- Scan parquet (126) - : +- Sort (137) - : +- Exchange (136) - : +- Filter (135) - : +- Scan parquet (134) - +- Sort (146) - +- Exchange (145) - +- Project (144) - +- Filter (143) - +- Scan parquet (142) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- SortMergeJoin Inner (137) + :- Sort (131) + : +- Exchange (130) + : +- Project (129) + : +- SortMergeJoin Inner (128) + : :- Sort (123) + : : +- Exchange (122) + : : +- Project (121) + : : +- SortMergeJoin Inner (120) + : : :- Sort (115) + : : : +- Exchange (114) + : : : +- Project (113) + : : : +- SortMergeJoin Inner (112) + : : : :- Sort (107) + : : : : +- Exchange (106) + : : : : +- Project (105) + : : : : +- SortMergeJoin Inner (104) + : : : : :- Sort (98) + : : : : : +- Exchange (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Sort (103) + : : : : +- Exchange (102) + : : : : +- Project (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- Sort (111) + : : : +- Exchange (110) + : : : +- Filter (109) + : : : +- Scan parquet (108) + : : +- Sort (119) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Sort (127) + : +- Exchange (126) + : +- Filter (125) + : +- Scan parquet (124) + +- Sort (136) + +- Exchange (135) + +- Project (134) + +- Filter (133) + +- Scan parquet (132) (1) Scan parquet @@ -165,628 +155,588 @@ Input [2]: [c_custkey#X, c_nationkey#X] (8) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) SortExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(18) SortExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(24) InputAdapter -Input [2]: [c_nationkey#X, o_orderkey#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(27) Scan parquet +(24) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(35) SortExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(41) InputAdapter -Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(42) InputIteratorTransformer +(37) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(43) SortExecTransformer +(38) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 -(44) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(50) InputAdapter -Input [2]: [s_suppkey#X, s_nationkey#X] - -(51) InputIteratorTransformer +(45) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(52) SortExecTransformer +(46) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(55) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(56) ColumnarExchange +(50) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(58) InputAdapter +(52) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(59) InputIteratorTransformer +(53) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(60) SortExecTransformer -Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(61) Scan parquet +(54) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(62) NoopFilter +(55) NoopFilter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(60) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] - -(69) SortExecTransformer +(61) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(75) InputAdapter -Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] - -(76) InputIteratorTransformer +(67) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(77) SortExecTransformer +(68) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 -(78) Scan parquet +(69) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(79) NoopFilter +(70) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(81) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(82) ColumnarExchange +(73) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(74) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(84) InputAdapter -Input [1]: [r_regionkey#X] - -(85) InputIteratorTransformer +(75) InputAdapter Input [1]: [r_regionkey#X] -(86) SortExecTransformer +(76) InputIteratorTransformer Input [1]: [r_regionkey#X] -Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 -(87) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(88) ProjectExecTransformer +(78) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(89) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(90) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(91) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(82) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(83) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(84) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(85) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(97) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(98) ColumnarExchange +(88) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(99) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(100) InputAdapter +(90) InputAdapter Input [2]: [n_name#X, revenue#X] -(101) InputIteratorTransformer +(91) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(102) SortExecTransformer +(92) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(103) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(104) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(105) Scan parquet +(95) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(106) Filter +(96) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(107) Exchange +(97) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Sort +(98) Sort Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(109) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(110) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(111) Project +(101) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(112) Exchange +(102) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Sort +(103) Sort Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(114) SortMergeJoin +(104) SortMergeJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(115) Project +(105) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(116) Exchange +(106) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Sort +(107) Sort Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(118) Scan parquet +(108) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(119) Filter +(109) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(120) Exchange +(110) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) Sort +(111) Sort Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(122) SortMergeJoin +(112) SortMergeJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(123) Project +(113) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(124) Exchange +(114) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(115) Sort Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_suppkey#X ASC NULLS FIRST, c_nationkey#X ASC NULLS FIRST], false, 0 -(126) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(127) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(128) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(119) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST, s_nationkey#X ASC NULLS FIRST], false, 0 -(130) SortMergeJoin +(120) SortMergeJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(131) Project +(121) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(132) Exchange +(122) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(123) Sort Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(134) Scan parquet +(124) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(135) Filter +(125) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(136) Exchange +(126) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) Sort +(127) Sort Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(138) SortMergeJoin +(128) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(139) Project +(129) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(140) Exchange +(130) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) Sort +(131) Sort Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 -(142) Scan parquet +(132) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(143) Filter +(133) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(144) Project +(134) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(145) Exchange +(135) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(146) Sort +(136) Sort Input [1]: [r_regionkey#X] Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 -(147) SortMergeJoin +(137) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(148) Project +(138) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(149) HashAggregate +(139) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(150) Exchange +(140) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) HashAggregate +(141) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(152) Exchange +(142) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(153) Sort +(143) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(154) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt index 00dc2897c578..e902306a960b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt @@ -1,130 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (148) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (100) - +- ^ SortExecTransformer (98) - +- ^ InputIteratorTransformer (97) - +- ShuffleQueryStage (95), Statistics(X) - +- ColumnarExchange (94) - +- ^ RegularHashAggregateExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ ProjectExecTransformer (86) - +- ^ FlushableHashAggregateExecTransformer (85) - +- ^ ProjectExecTransformer (84) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (83) - :- ^ SortExecTransformer (77) - : +- ^ InputIteratorTransformer (76) - : +- ShuffleQueryStage (74), Statistics(X) - : +- ColumnarExchange (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - : :- ^ SortExecTransformer (60) - : : +- ^ InputIteratorTransformer (59) - : : +- ShuffleQueryStage (57), Statistics(X) - : : +- ColumnarExchange (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) - : : :- ^ SortExecTransformer (43) - : : : +- ^ InputIteratorTransformer (42) - : : : +- ShuffleQueryStage (40), Statistics(X) - : : : +- ColumnarExchange (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ SortExecTransformer (26) - : : : : +- ^ InputIteratorTransformer (25) - : : : : +- ShuffleQueryStage (23), Statistics(X) - : : : : +- ColumnarExchange (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ SortExecTransformer (9) - : : : : : +- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ SortExecTransformer (18) - : : : : +- ^ InputIteratorTransformer (17) - : : : : +- ShuffleQueryStage (15), Statistics(X) - : : : : +- ColumnarExchange (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ SortExecTransformer (35) - : : : +- ^ InputIteratorTransformer (34) - : : : +- ShuffleQueryStage (32), Statistics(X) - : : : +- ColumnarExchange (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ SortExecTransformer (52) - : : +- ^ InputIteratorTransformer (51) - : : +- ShuffleQueryStage (49), Statistics(X) - : : +- ColumnarExchange (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ReusedExchange (78) + VeloxColumnarToRowExec (90) + +- ^ SortExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ RegularHashAggregateExecTransformer (82) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ColumnarExchange (78) + +- ^ ProjectExecTransformer (76) + +- ^ FlushableHashAggregateExecTransformer (75) + +- ^ ProjectExecTransformer (74) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (72) + +- ShuffleQueryStage (70), Statistics(X) + +- ReusedExchange (69) +- == Initial Plan == - Sort (147) - +- Exchange (146) - +- HashAggregate (145) - +- Exchange (144) - +- HashAggregate (143) - +- Project (142) - +- SortMergeJoin Inner (141) - :- Sort (136) - : +- Exchange (135) - : +- Project (134) - : +- SortMergeJoin Inner (133) - : :- Sort (128) - : : +- Exchange (127) - : : +- Project (126) - : : +- SortMergeJoin Inner (125) - : : :- Sort (120) - : : : +- Exchange (119) - : : : +- Project (118) - : : : +- SortMergeJoin Inner (117) - : : : :- Sort (112) - : : : : +- Exchange (111) - : : : : +- Project (110) - : : : : +- SortMergeJoin Inner (109) - : : : : :- Sort (104) - : : : : : +- Exchange (103) - : : : : : +- Filter (102) - : : : : : +- Scan parquet (101) - : : : : +- Sort (108) - : : : : +- Exchange (107) - : : : : +- Filter (106) - : : : : +- Scan parquet (105) - : : : +- Sort (116) - : : : +- Exchange (115) - : : : +- Filter (114) - : : : +- Scan parquet (113) - : : +- Sort (124) - : : +- Exchange (123) - : : +- Filter (122) - : : +- Scan parquet (121) - : +- Sort (132) - : +- Exchange (131) - : +- Filter (130) - : +- Scan parquet (129) - +- Sort (140) - +- Exchange (139) - +- Filter (138) - +- Scan parquet (137) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- SortMergeJoin Inner (131) + :- Sort (126) + : +- Exchange (125) + : +- Project (124) + : +- SortMergeJoin Inner (123) + : :- Sort (118) + : : +- Exchange (117) + : : +- Project (116) + : : +- SortMergeJoin Inner (115) + : : :- Sort (110) + : : : +- Exchange (109) + : : : +- Project (108) + : : : +- SortMergeJoin Inner (107) + : : : :- Sort (102) + : : : : +- Exchange (101) + : : : : +- Project (100) + : : : : +- SortMergeJoin Inner (99) + : : : : :- Sort (94) + : : : : : +- Exchange (93) + : : : : : +- Filter (92) + : : : : : +- Scan parquet (91) + : : : : +- Sort (98) + : : : : +- Exchange (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Sort (106) + : : : +- Exchange (105) + : : : +- Filter (104) + : : : +- Scan parquet (103) + : : +- Sort (114) + : : +- Exchange (113) + : : +- Filter (112) + : : +- Scan parquet (111) + : +- Sort (122) + : +- Exchange (121) + : +- Filter (120) + : +- Scan parquet (119) + +- Sort (130) + +- Exchange (129) + +- Filter (128) + +- Scan parquet (127) (1) Scan parquet @@ -160,600 +150,560 @@ Input [2]: [s_suppkey#X, s_nationkey#X] (8) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) SortExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(18) SortExecTransformer -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) SortExecTransformer -Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 - -(27) Scan parquet +(24) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X, o_custkey#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(35) SortExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(41) InputAdapter +(37) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(42) InputIteratorTransformer +(38) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(43) SortExecTransformer -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(44) Scan parquet +(39) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(50) InputAdapter +(45) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(51) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(52) SortExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 - -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(55) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(56) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(58) InputAdapter +(52) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(59) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(60) SortExecTransformer -Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(61) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(62) NoopFilter +(55) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(68) InputIteratorTransformer +(60) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(69) SortExecTransformer +(61) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter -Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] - -(76) InputIteratorTransformer +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(77) SortExecTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(78) ReusedExchange [Reuses operator id: 65] +(69) ReusedExchange [Reuses operator id: 58] Output [2]: [n_nationkey#X, n_name#X] -(79) ShuffleQueryStage +(70) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(80) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(81) InputIteratorTransformer +(71) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(82) SortExecTransformer +(72) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(83) ShuffledHashJoinExecTransformer +(73) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(84) ProjectExecTransformer +(74) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(85) FlushableHashAggregateExecTransformer +(75) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(86) ProjectExecTransformer +(76) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(87) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(88) ColumnarExchange +(78) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(79) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(90) InputAdapter +(80) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(91) InputIteratorTransformer +(81) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(92) RegularHashAggregateExecTransformer +(82) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(93) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(94) ColumnarExchange +(84) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(95) ShuffleQueryStage +(85) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(96) InputAdapter +(86) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(97) InputIteratorTransformer +(87) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(98) SortExecTransformer +(88) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(99) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(100) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(101) Scan parquet +(91) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(102) Filter +(92) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(103) Exchange +(93) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) Sort +(94) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(105) Scan parquet +(95) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(106) Filter +(96) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(107) Exchange +(97) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Sort +(98) Sort Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(109) SortMergeJoin +(99) SortMergeJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(110) Project +(100) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(111) Exchange +(101) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Sort +(102) Sort Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(113) Scan parquet +(103) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(114) Filter +(104) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(115) Exchange +(105) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) Sort +(106) Sort Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(117) SortMergeJoin +(107) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(118) Project +(108) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(119) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) Sort +(110) Sort Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(121) Scan parquet +(111) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(122) Filter +(112) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(123) Exchange +(113) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(124) Sort +(114) Sort Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(125) SortMergeJoin +(115) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(126) Project +(116) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(127) Exchange +(117) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(118) Sort Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(129) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(130) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(131) Exchange +(121) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(122) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(133) SortMergeJoin +(123) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(134) Project +(124) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(135) Exchange +(125) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Sort +(126) Sort Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(137) Scan parquet +(127) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(138) Filter +(128) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(139) Exchange +(129) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(140) Sort +(130) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(141) SortMergeJoin +(131) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(142) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(143) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(144) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(146) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(147) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(148) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt index 16aa6bb4330f..65b83c6e0fa3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt @@ -1,178 +1,164 @@ == Physical Plan == -AdaptiveSparkPlan (205) +AdaptiveSparkPlan (191) +- == Final Plan == - VeloxColumnarToRowExec (139) - +- ^ SortExecTransformer (137) - +- ^ InputIteratorTransformer (136) - +- ShuffleQueryStage (134), Statistics(X) - +- ColumnarExchange (133) - +- ^ ProjectExecTransformer (131) - +- ^ RegularHashAggregateExecTransformer (130) - +- ^ InputIteratorTransformer (129) - +- ShuffleQueryStage (127), Statistics(X) - +- ColumnarExchange (126) - +- ^ ProjectExecTransformer (124) - +- ^ FlushableHashAggregateExecTransformer (123) - +- ^ ProjectExecTransformer (122) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) - :- ^ SortExecTransformer (111) - : +- ^ InputIteratorTransformer (110) - : +- ShuffleQueryStage (108), Statistics(X) - : +- ColumnarExchange (107) - : +- ^ ProjectExecTransformer (105) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) - : :- ^ SortExecTransformer (94) - : : +- ^ InputIteratorTransformer (93) - : : +- ShuffleQueryStage (91), Statistics(X) - : : +- ColumnarExchange (90) - : : +- ^ ProjectExecTransformer (88) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - : : :- ^ SortExecTransformer (77) - : : : +- ^ InputIteratorTransformer (76) - : : : +- ShuffleQueryStage (74), Statistics(X) - : : : +- ColumnarExchange (73) - : : : +- ^ ProjectExecTransformer (71) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : : : :- ^ SortExecTransformer (60) - : : : : +- ^ InputIteratorTransformer (59) - : : : : +- ShuffleQueryStage (57), Statistics(X) - : : : : +- ColumnarExchange (56) - : : : : +- ^ ProjectExecTransformer (54) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : : : :- ^ SortExecTransformer (43) - : : : : : +- ^ InputIteratorTransformer (42) - : : : : : +- ShuffleQueryStage (40), Statistics(X) - : : : : : +- ColumnarExchange (39) - : : : : : +- ^ ProjectExecTransformer (37) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : : : :- ^ SortExecTransformer (26) - : : : : : : +- ^ InputIteratorTransformer (25) - : : : : : : +- ShuffleQueryStage (23), Statistics(X) - : : : : : : +- ColumnarExchange (22) - : : : : : : +- ^ ProjectExecTransformer (20) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : : : :- ^ SortExecTransformer (9) - : : : : : : : +- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ SortExecTransformer (18) - : : : : : : +- ^ InputIteratorTransformer (17) - : : : : : : +- ShuffleQueryStage (15), Statistics(X) - : : : : : : +- ColumnarExchange (14) - : : : : : : +- ^ ProjectExecTransformer (12) - : : : : : : +- ^ NoopFilter (11) - : : : : : : +- ^ Scan parquet (10) - : : : : : +- ^ SortExecTransformer (35) - : : : : : +- ^ InputIteratorTransformer (34) - : : : : : +- ShuffleQueryStage (32), Statistics(X) - : : : : : +- ColumnarExchange (31) - : : : : : +- ^ ProjectExecTransformer (29) - : : : : : +- ^ NoopFilter (28) - : : : : : +- ^ Scan parquet (27) - : : : : +- ^ SortExecTransformer (52) - : : : : +- ^ InputIteratorTransformer (51) - : : : : +- ShuffleQueryStage (49), Statistics(X) - : : : : +- ColumnarExchange (48) - : : : : +- ^ ProjectExecTransformer (46) - : : : : +- ^ NoopFilter (45) - : : : : +- ^ Scan parquet (44) - : : : +- ^ SortExecTransformer (69) - : : : +- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ NoopFilter (62) - : : : +- ^ Scan parquet (61) - : : +- ^ SortExecTransformer (86) - : : +- ^ InputIteratorTransformer (85) - : : +- ShuffleQueryStage (83), Statistics(X) - : : +- ColumnarExchange (82) - : : +- ^ ProjectExecTransformer (80) - : : +- ^ NoopFilter (79) - : : +- ^ Scan parquet (78) - : +- ^ SortExecTransformer (103) - : +- ^ InputIteratorTransformer (102) - : +- ShuffleQueryStage (100), Statistics(X) - : +- ColumnarExchange (99) - : +- ^ ProjectExecTransformer (97) - : +- ^ NoopFilter (96) - : +- ^ Scan parquet (95) - +- ^ SortExecTransformer (120) - +- ^ InputIteratorTransformer (119) - +- ShuffleQueryStage (117), Statistics(X) - +- ColumnarExchange (116) - +- ^ ProjectExecTransformer (114) - +- ^ NoopFilter (113) - +- ^ Scan parquet (112) + VeloxColumnarToRowExec (125) + +- ^ SortExecTransformer (123) + +- ^ InputIteratorTransformer (122) + +- ShuffleQueryStage (120), Statistics(X) + +- ColumnarExchange (119) + +- ^ ProjectExecTransformer (117) + +- ^ RegularHashAggregateExecTransformer (116) + +- ^ InputIteratorTransformer (115) + +- ShuffleQueryStage (113), Statistics(X) + +- ColumnarExchange (112) + +- ^ ProjectExecTransformer (110) + +- ^ FlushableHashAggregateExecTransformer (109) + +- ^ ProjectExecTransformer (108) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- ^ ProjectExecTransformer (93) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) + : :- ^ InputIteratorTransformer (83) + : : +- ShuffleQueryStage (81), Statistics(X) + : : +- ColumnarExchange (80) + : : +- ^ ProjectExecTransformer (78) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + : : :- ^ InputIteratorTransformer (68) + : : : +- ShuffleQueryStage (66), Statistics(X) + : : : +- ColumnarExchange (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : : : :- ^ InputIteratorTransformer (53) + : : : : +- ShuffleQueryStage (51), Statistics(X) + : : : : +- ColumnarExchange (50) + : : : : +- ^ ProjectExecTransformer (48) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : : : :- ^ InputIteratorTransformer (38) + : : : : : +- ShuffleQueryStage (36), Statistics(X) + : : : : : +- ColumnarExchange (35) + : : : : : +- ^ ProjectExecTransformer (33) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : : : :- ^ InputIteratorTransformer (23) + : : : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : : : +- ColumnarExchange (20) + : : : : : : +- ^ ProjectExecTransformer (18) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (16) + : : : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : : : +- ColumnarExchange (13) + : : : : : : +- ^ ProjectExecTransformer (11) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (31) + : : : : : +- ShuffleQueryStage (29), Statistics(X) + : : : : : +- ColumnarExchange (28) + : : : : : +- ^ ProjectExecTransformer (26) + : : : : : +- ^ NoopFilter (25) + : : : : : +- ^ Scan parquet (24) + : : : : +- ^ InputIteratorTransformer (46) + : : : : +- ShuffleQueryStage (44), Statistics(X) + : : : : +- ColumnarExchange (43) + : : : : +- ^ ProjectExecTransformer (41) + : : : : +- ^ NoopFilter (40) + : : : : +- ^ Scan parquet (39) + : : : +- ^ InputIteratorTransformer (61) + : : : +- ShuffleQueryStage (59), Statistics(X) + : : : +- ColumnarExchange (58) + : : : +- ^ ProjectExecTransformer (56) + : : : +- ^ NoopFilter (55) + : : : +- ^ Scan parquet (54) + : : +- ^ InputIteratorTransformer (76) + : : +- ShuffleQueryStage (74), Statistics(X) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ NoopFilter (70) + : : +- ^ Scan parquet (69) + : +- ^ InputIteratorTransformer (91) + : +- ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ NoopFilter (85) + : +- ^ Scan parquet (84) + +- ^ InputIteratorTransformer (106) + +- ShuffleQueryStage (104), Statistics(X) + +- ColumnarExchange (103) + +- ^ ProjectExecTransformer (101) + +- ^ NoopFilter (100) + +- ^ Scan parquet (99) +- == Initial Plan == - Sort (204) - +- Exchange (203) - +- HashAggregate (202) - +- Exchange (201) - +- HashAggregate (200) - +- Project (199) - +- SortMergeJoin Inner (198) - :- Sort (192) - : +- Exchange (191) - : +- Project (190) - : +- SortMergeJoin Inner (189) - : :- Sort (184) - : : +- Exchange (183) - : : +- Project (182) - : : +- SortMergeJoin Inner (181) - : : :- Sort (176) - : : : +- Exchange (175) - : : : +- Project (174) - : : : +- SortMergeJoin Inner (173) - : : : :- Sort (168) - : : : : +- Exchange (167) - : : : : +- Project (166) - : : : : +- SortMergeJoin Inner (165) - : : : : :- Sort (160) - : : : : : +- Exchange (159) - : : : : : +- Project (158) - : : : : : +- SortMergeJoin Inner (157) - : : : : : :- Sort (152) - : : : : : : +- Exchange (151) - : : : : : : +- Project (150) - : : : : : : +- SortMergeJoin Inner (149) - : : : : : : :- Sort (144) - : : : : : : : +- Exchange (143) - : : : : : : : +- Project (142) - : : : : : : : +- Filter (141) - : : : : : : : +- Scan parquet (140) - : : : : : : +- Sort (148) - : : : : : : +- Exchange (147) - : : : : : : +- Filter (146) - : : : : : : +- Scan parquet (145) - : : : : : +- Sort (156) - : : : : : +- Exchange (155) - : : : : : +- Filter (154) - : : : : : +- Scan parquet (153) - : : : : +- Sort (164) - : : : : +- Exchange (163) - : : : : +- Filter (162) - : : : : +- Scan parquet (161) - : : : +- Sort (172) - : : : +- Exchange (171) - : : : +- Filter (170) - : : : +- Scan parquet (169) - : : +- Sort (180) - : : +- Exchange (179) - : : +- Filter (178) - : : +- Scan parquet (177) - : +- Sort (188) - : +- Exchange (187) - : +- Filter (186) - : +- Scan parquet (185) - +- Sort (197) - +- Exchange (196) - +- Project (195) - +- Filter (194) - +- Scan parquet (193) + Sort (190) + +- Exchange (189) + +- HashAggregate (188) + +- Exchange (187) + +- HashAggregate (186) + +- Project (185) + +- SortMergeJoin Inner (184) + :- Sort (178) + : +- Exchange (177) + : +- Project (176) + : +- SortMergeJoin Inner (175) + : :- Sort (170) + : : +- Exchange (169) + : : +- Project (168) + : : +- SortMergeJoin Inner (167) + : : :- Sort (162) + : : : +- Exchange (161) + : : : +- Project (160) + : : : +- SortMergeJoin Inner (159) + : : : :- Sort (154) + : : : : +- Exchange (153) + : : : : +- Project (152) + : : : : +- SortMergeJoin Inner (151) + : : : : :- Sort (146) + : : : : : +- Exchange (145) + : : : : : +- Project (144) + : : : : : +- SortMergeJoin Inner (143) + : : : : : :- Sort (138) + : : : : : : +- Exchange (137) + : : : : : : +- Project (136) + : : : : : : +- SortMergeJoin Inner (135) + : : : : : : :- Sort (130) + : : : : : : : +- Exchange (129) + : : : : : : : +- Project (128) + : : : : : : : +- Filter (127) + : : : : : : : +- Scan parquet (126) + : : : : : : +- Sort (134) + : : : : : : +- Exchange (133) + : : : : : : +- Filter (132) + : : : : : : +- Scan parquet (131) + : : : : : +- Sort (142) + : : : : : +- Exchange (141) + : : : : : +- Filter (140) + : : : : : +- Scan parquet (139) + : : : : +- Sort (150) + : : : : +- Exchange (149) + : : : : +- Filter (148) + : : : : +- Scan parquet (147) + : : : +- Sort (158) + : : : +- Exchange (157) + : : : +- Filter (156) + : : : +- Scan parquet (155) + : : +- Sort (166) + : : +- Exchange (165) + : : +- Filter (164) + : : +- Scan parquet (163) + : +- Sort (174) + : +- Exchange (173) + : +- Filter (172) + : +- Scan parquet (171) + +- Sort (183) + +- Exchange (182) + +- Project (181) + +- Filter (180) + +- Scan parquet (179) (1) Scan parquet @@ -208,844 +194,788 @@ Input [1]: [p_partkey#X] (8) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) SortExecTransformer -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(18) SortExecTransformer -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(24) InputAdapter -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(27) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(35) SortExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(41) InputAdapter -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(42) InputIteratorTransformer +(37) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(43) SortExecTransformer +(38) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(44) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(50) InputAdapter -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] - -(51) InputIteratorTransformer +(45) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(52) SortExecTransformer +(46) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(55) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(56) ColumnarExchange +(50) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(58) InputAdapter +(52) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(59) InputIteratorTransformer +(53) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(60) SortExecTransformer -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 - -(61) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) NoopFilter +(55) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(60) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(68) InputIteratorTransformer -Input [2]: [c_custkey#X, c_nationkey#X] - -(69) SortExecTransformer +(61) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(75) InputAdapter -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] - -(76) InputIteratorTransformer +(67) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(77) SortExecTransformer +(68) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(78) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(79) NoopFilter +(70) NoopFilter Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X, n_regionkey#X] -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(81) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(82) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(84) InputAdapter -Input [2]: [n_nationkey#X, n_regionkey#X] - -(85) InputIteratorTransformer +(75) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(86) SortExecTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(87) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) ProjectExecTransformer +(78) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(89) WholeStageCodegenTransformer (X) +(79) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(90) ColumnarExchange +(80) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(91) ShuffleQueryStage +(81) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(92) InputAdapter +(82) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(93) InputIteratorTransformer +(83) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(94) SortExecTransformer -Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 - -(95) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(96) NoopFilter +(85) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(97) ProjectExecTransformer +(86) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(98) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(99) ColumnarExchange +(88) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(100) ShuffleQueryStage +(89) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(101) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(102) InputIteratorTransformer +(90) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(103) SortExecTransformer +(91) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(104) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(105) ProjectExecTransformer +(93) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(106) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(107) ColumnarExchange +(95) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(108) ShuffleQueryStage +(96) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(109) InputAdapter +(97) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(110) InputIteratorTransformer +(98) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(111) SortExecTransformer -Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 - -(112) Scan parquet +(99) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(113) NoopFilter +(100) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(114) ProjectExecTransformer +(101) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(115) WholeStageCodegenTransformer (X) +(102) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(116) ColumnarExchange +(103) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(117) ShuffleQueryStage +(104) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(118) InputAdapter -Input [1]: [r_regionkey#X] - -(119) InputIteratorTransformer +(105) InputAdapter Input [1]: [r_regionkey#X] -(120) SortExecTransformer +(106) InputIteratorTransformer Input [1]: [r_regionkey#X] -Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 -(121) ShuffledHashJoinExecTransformer +(107) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(122) ProjectExecTransformer +(108) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(123) FlushableHashAggregateExecTransformer +(109) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(124) ProjectExecTransformer +(110) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) WholeStageCodegenTransformer (X) +(111) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(126) ColumnarExchange +(112) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(127) ShuffleQueryStage +(113) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(128) InputAdapter +(114) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(129) InputIteratorTransformer +(115) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(130) RegularHashAggregateExecTransformer +(116) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(131) ProjectExecTransformer +(117) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(132) WholeStageCodegenTransformer (X) +(118) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(133) ColumnarExchange +(119) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(134) ShuffleQueryStage +(120) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(135) InputAdapter +(121) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(136) InputIteratorTransformer +(122) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(137) SortExecTransformer +(123) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(138) WholeStageCodegenTransformer (X) +(124) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(139) VeloxColumnarToRowExec +(125) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(140) Scan parquet +(126) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(141) Filter +(127) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(142) Project +(128) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(143) Exchange +(129) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(144) Sort +(130) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(145) Scan parquet +(131) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(146) Filter +(132) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(147) Exchange +(133) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Sort +(134) Sort Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(149) SortMergeJoin +(135) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(150) Project +(136) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(151) Exchange +(137) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Sort +(138) Sort Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(153) Scan parquet +(139) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(154) Filter +(140) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(155) Exchange +(141) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(156) Sort +(142) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(157) SortMergeJoin +(143) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(158) Project +(144) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(159) Exchange +(145) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Sort +(146) Sort Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(161) Scan parquet +(147) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(162) Filter +(148) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(163) Exchange +(149) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(164) Sort +(150) Sort Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(165) SortMergeJoin +(151) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(166) Project +(152) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(167) Exchange +(153) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(168) Sort +(154) Sort Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_custkey#X ASC NULLS FIRST], false, 0 -(169) Scan parquet +(155) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(170) Filter +(156) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(171) Exchange +(157) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(172) Sort +(158) Sort Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X ASC NULLS FIRST], false, 0 -(173) SortMergeJoin +(159) SortMergeJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(174) Project +(160) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(175) Exchange +(161) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(162) Sort Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: [c_nationkey#X ASC NULLS FIRST], false, 0 -(177) Scan parquet +(163) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(178) Filter +(164) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(179) Exchange +(165) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(180) Sort +(166) Sort Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(181) SortMergeJoin +(167) SortMergeJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(182) Project +(168) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(183) Exchange +(169) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(184) Sort +(170) Sort Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(185) Scan parquet +(171) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(186) Filter +(172) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(187) Exchange +(173) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(188) Sort +(174) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(189) SortMergeJoin +(175) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(190) Project +(176) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(191) Exchange +(177) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(192) Sort +(178) Sort Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: [n_regionkey#X ASC NULLS FIRST], false, 0 -(193) Scan parquet +(179) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(194) Filter +(180) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(195) Project +(181) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(196) Exchange +(182) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(197) Sort +(183) Sort Input [1]: [r_regionkey#X] Arguments: [r_regionkey#X ASC NULLS FIRST], false, 0 -(198) SortMergeJoin +(184) SortMergeJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(199) Project +(185) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(200) HashAggregate +(186) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(201) Exchange +(187) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(202) HashAggregate +(188) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(203) Exchange +(189) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(204) Sort +(190) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(205) AdaptiveSparkPlan +(191) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt index 183c1a378ba6..77b442accef4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt @@ -1,134 +1,124 @@ == Physical Plan == -AdaptiveSparkPlan (153) +AdaptiveSparkPlan (143) +- == Final Plan == - VeloxColumnarToRowExec (104) - +- ^ SortExecTransformer (102) - +- ^ InputIteratorTransformer (101) - +- ShuffleQueryStage (99), Statistics(X) - +- ColumnarExchange (98) - +- ^ RegularHashAggregateExecTransformer (96) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ColumnarExchange (92) - +- ^ ProjectExecTransformer (90) - +- ^ FlushableHashAggregateExecTransformer (89) - +- ^ ProjectExecTransformer (88) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) - :- ^ SortExecTransformer (77) - : +- ^ InputIteratorTransformer (76) - : +- ShuffleQueryStage (74), Statistics(X) - : +- ColumnarExchange (73) - : +- ^ ProjectExecTransformer (71) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) - : :- ^ SortExecTransformer (60) - : : +- ^ InputIteratorTransformer (59) - : : +- ShuffleQueryStage (57), Statistics(X) - : : +- ColumnarExchange (56) - : : +- ^ ProjectExecTransformer (54) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) - : : :- ^ SortExecTransformer (43) - : : : +- ^ InputIteratorTransformer (42) - : : : +- ShuffleQueryStage (40), Statistics(X) - : : : +- ColumnarExchange (39) - : : : +- ^ ProjectExecTransformer (37) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) - : : : :- ^ SortExecTransformer (26) - : : : : +- ^ InputIteratorTransformer (25) - : : : : +- ShuffleQueryStage (23), Statistics(X) - : : : : +- ColumnarExchange (22) - : : : : +- ^ ProjectExecTransformer (20) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) - : : : : :- ^ SortExecTransformer (9) - : : : : : +- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ SortExecTransformer (18) - : : : : +- ^ InputIteratorTransformer (17) - : : : : +- ShuffleQueryStage (15), Statistics(X) - : : : : +- ColumnarExchange (14) - : : : : +- ^ ProjectExecTransformer (12) - : : : : +- ^ NoopFilter (11) - : : : : +- ^ Scan parquet (10) - : : : +- ^ SortExecTransformer (35) - : : : +- ^ InputIteratorTransformer (34) - : : : +- ShuffleQueryStage (32), Statistics(X) - : : : +- ColumnarExchange (31) - : : : +- ^ ProjectExecTransformer (29) - : : : +- ^ NoopFilter (28) - : : : +- ^ Scan parquet (27) - : : +- ^ SortExecTransformer (52) - : : +- ^ InputIteratorTransformer (51) - : : +- ShuffleQueryStage (49), Statistics(X) - : : +- ColumnarExchange (48) - : : +- ^ ProjectExecTransformer (46) - : : +- ^ NoopFilter (45) - : : +- ^ Scan parquet (44) - : +- ^ SortExecTransformer (69) - : +- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ NoopFilter (62) - : +- ^ Scan parquet (61) - +- ^ SortExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ NoopFilter (79) - +- ^ Scan parquet (78) + VeloxColumnarToRowExec (94) + +- ^ SortExecTransformer (92) + +- ^ InputIteratorTransformer (91) + +- ShuffleQueryStage (89), Statistics(X) + +- ColumnarExchange (88) + +- ^ RegularHashAggregateExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ ProjectExecTransformer (80) + +- ^ FlushableHashAggregateExecTransformer (79) + +- ^ ProjectExecTransformer (78) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) + :- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) + : :- ^ InputIteratorTransformer (53) + : : +- ShuffleQueryStage (51), Statistics(X) + : : +- ColumnarExchange (50) + : : +- ^ ProjectExecTransformer (48) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) + : : :- ^ InputIteratorTransformer (38) + : : : +- ShuffleQueryStage (36), Statistics(X) + : : : +- ColumnarExchange (35) + : : : +- ^ ProjectExecTransformer (33) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) + : : : :- ^ InputIteratorTransformer (23) + : : : : +- ShuffleQueryStage (21), Statistics(X) + : : : : +- ColumnarExchange (20) + : : : : +- ^ ProjectExecTransformer (18) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- ShuffleQueryStage (6), Statistics(X) + : : : : : +- ColumnarExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (16) + : : : : +- ShuffleQueryStage (14), Statistics(X) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (31) + : : : +- ShuffleQueryStage (29), Statistics(X) + : : : +- ColumnarExchange (28) + : : : +- ^ ProjectExecTransformer (26) + : : : +- ^ NoopFilter (25) + : : : +- ^ Scan parquet (24) + : : +- ^ InputIteratorTransformer (46) + : : +- ShuffleQueryStage (44), Statistics(X) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ NoopFilter (40) + : : +- ^ Scan parquet (39) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ NoopFilter (55) + : +- ^ Scan parquet (54) + +- ^ InputIteratorTransformer (76) + +- ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ NoopFilter (70) + +- ^ Scan parquet (69) +- == Initial Plan == - Sort (152) - +- Exchange (151) - +- HashAggregate (150) - +- Exchange (149) - +- HashAggregate (148) - +- Project (147) - +- SortMergeJoin Inner (146) - :- Sort (141) - : +- Exchange (140) - : +- Project (139) - : +- SortMergeJoin Inner (138) - : :- Sort (133) - : : +- Exchange (132) - : : +- Project (131) - : : +- SortMergeJoin Inner (130) - : : :- Sort (125) - : : : +- Exchange (124) - : : : +- Project (123) - : : : +- SortMergeJoin Inner (122) - : : : :- Sort (117) - : : : : +- Exchange (116) - : : : : +- Project (115) - : : : : +- SortMergeJoin Inner (114) - : : : : :- Sort (109) - : : : : : +- Exchange (108) - : : : : : +- Project (107) - : : : : : +- Filter (106) - : : : : : +- Scan parquet (105) - : : : : +- Sort (113) - : : : : +- Exchange (112) - : : : : +- Filter (111) - : : : : +- Scan parquet (110) - : : : +- Sort (121) - : : : +- Exchange (120) - : : : +- Filter (119) - : : : +- Scan parquet (118) - : : +- Sort (129) - : : +- Exchange (128) - : : +- Filter (127) - : : +- Scan parquet (126) - : +- Sort (137) - : +- Exchange (136) - : +- Filter (135) - : +- Scan parquet (134) - +- Sort (145) - +- Exchange (144) - +- Filter (143) - +- Scan parquet (142) + Sort (142) + +- Exchange (141) + +- HashAggregate (140) + +- Exchange (139) + +- HashAggregate (138) + +- Project (137) + +- SortMergeJoin Inner (136) + :- Sort (131) + : +- Exchange (130) + : +- Project (129) + : +- SortMergeJoin Inner (128) + : :- Sort (123) + : : +- Exchange (122) + : : +- Project (121) + : : +- SortMergeJoin Inner (120) + : : :- Sort (115) + : : : +- Exchange (114) + : : : +- Project (113) + : : : +- SortMergeJoin Inner (112) + : : : :- Sort (107) + : : : : +- Exchange (106) + : : : : +- Project (105) + : : : : +- SortMergeJoin Inner (104) + : : : : :- Sort (99) + : : : : : +- Exchange (98) + : : : : : +- Project (97) + : : : : : +- Filter (96) + : : : : : +- Scan parquet (95) + : : : : +- Sort (103) + : : : : +- Exchange (102) + : : : : +- Filter (101) + : : : : +- Scan parquet (100) + : : : +- Sort (111) + : : : +- Exchange (110) + : : : +- Filter (109) + : : : +- Scan parquet (108) + : : +- Sort (119) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Sort (127) + : +- Exchange (126) + : +- Filter (125) + : +- Scan parquet (124) + +- Sort (135) + +- Exchange (134) + +- Filter (133) + +- Scan parquet (132) (1) Scan parquet @@ -164,624 +154,584 @@ Input [1]: [p_partkey#X] (8) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) SortExecTransformer -Input [1]: [p_partkey#X] -Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 - -(10) Scan parquet +(9) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(11) NoopFilter +(10) NoopFilter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(18) SortExecTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 - -(19) ShuffledHashJoinExecTransformer +(17) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(24) InputAdapter -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] - -(25) InputIteratorTransformer +(22) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(26) SortExecTransformer +(23) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(27) Scan parquet +(24) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(28) NoopFilter +(25) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(30) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(31) ColumnarExchange +(28) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(29) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(35) SortExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 - -(36) ShuffledHashJoinExecTransformer +(32) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(37) ProjectExecTransformer +(33) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(38) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(39) ColumnarExchange +(35) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(40) ShuffleQueryStage +(36) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(41) InputAdapter -Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] - -(42) InputIteratorTransformer +(37) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(43) SortExecTransformer +(38) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 -(44) Scan parquet +(39) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(45) NoopFilter +(40) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(50) InputAdapter +(45) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(51) InputIteratorTransformer +(46) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(52) SortExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 - -(53) ShuffledHashJoinExecTransformer +(47) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(54) ProjectExecTransformer +(48) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(55) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(56) ColumnarExchange +(50) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(57) ShuffleQueryStage +(51) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(58) InputAdapter -Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] - -(59) InputIteratorTransformer +(52) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(60) SortExecTransformer +(53) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(61) Scan parquet +(54) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(62) NoopFilter +(55) NoopFilter Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_orderdate#X] -(63) ProjectExecTransformer +(56) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(58) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(67) InputAdapter -Input [2]: [o_orderkey#X, o_orderdate#X] - -(68) InputIteratorTransformer +(60) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(69) SortExecTransformer +(61) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(70) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(71) ProjectExecTransformer +(63) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(72) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(73) ColumnarExchange +(65) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(66) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(75) InputAdapter -Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] - -(76) InputIteratorTransformer +(67) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(77) SortExecTransformer +(68) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(78) Scan parquet +(69) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) NoopFilter +(70) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(80) ProjectExecTransformer +(71) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(81) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(82) ColumnarExchange +(73) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(74) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(84) InputAdapter -Input [2]: [n_nationkey#X, n_name#X] - -(85) InputIteratorTransformer +(75) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(86) SortExecTransformer +(76) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(87) ShuffledHashJoinExecTransformer +(77) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) ProjectExecTransformer +(78) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(89) FlushableHashAggregateExecTransformer +(79) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(90) ProjectExecTransformer +(80) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(91) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(92) ColumnarExchange +(82) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(93) ShuffleQueryStage +(83) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(94) InputAdapter +(84) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(95) InputIteratorTransformer +(85) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(96) RegularHashAggregateExecTransformer +(86) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(97) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(98) ColumnarExchange +(88) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(99) ShuffleQueryStage +(89) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(100) InputAdapter +(90) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(101) InputIteratorTransformer +(91) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(102) SortExecTransformer +(92) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(103) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(104) VeloxColumnarToRowExec +(94) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(105) Scan parquet +(95) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(106) Filter +(96) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(107) Project +(97) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(108) Exchange +(98) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Sort +(99) Sort Input [1]: [p_partkey#X] Arguments: [p_partkey#X ASC NULLS FIRST], false, 0 -(110) Scan parquet +(100) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(111) Filter +(101) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(112) Exchange +(102) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) Sort +(103) Sort Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_partkey#X ASC NULLS FIRST], false, 0 -(114) SortMergeJoin +(104) SortMergeJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(115) Project +(105) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(116) Exchange +(106) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Sort +(107) Sort Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_suppkey#X ASC NULLS FIRST], false, 0 -(118) Scan parquet +(108) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(119) Filter +(109) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(120) Exchange +(110) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) Sort +(111) Sort Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X ASC NULLS FIRST], false, 0 -(122) SortMergeJoin +(112) SortMergeJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(123) Project +(113) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(124) Exchange +(114) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(115) Sort Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: [l_suppkey#X ASC NULLS FIRST, l_partkey#X ASC NULLS FIRST], false, 0 -(126) Scan parquet +(116) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(127) Filter +(117) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(128) Exchange +(118) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(129) Sort +(119) Sort Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_suppkey#X ASC NULLS FIRST, ps_partkey#X ASC NULLS FIRST], false, 0 -(130) SortMergeJoin +(120) SortMergeJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(131) Project +(121) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(132) Exchange +(122) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(123) Sort Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: [l_orderkey#X ASC NULLS FIRST], false, 0 -(134) Scan parquet +(124) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(135) Filter +(125) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(136) Exchange +(126) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(137) Sort +(127) Sort Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X ASC NULLS FIRST], false, 0 -(138) SortMergeJoin +(128) SortMergeJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(139) Project +(129) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(140) Exchange +(130) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(141) Sort +(131) Sort Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: [s_nationkey#X ASC NULLS FIRST], false, 0 -(142) Scan parquet +(132) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(143) Filter +(133) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(144) Exchange +(134) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) Sort +(135) Sort Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X ASC NULLS FIRST], false, 0 -(146) SortMergeJoin +(136) SortMergeJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(147) Project +(137) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(148) HashAggregate +(138) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(149) Exchange +(139) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(150) HashAggregate +(140) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(151) Exchange +(141) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(152) Sort +(142) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(153) AdaptiveSparkPlan +(143) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/shims/common/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCost.scala b/shims/common/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCost.scala new file mode 100644 index 000000000000..9436f469ba79 --- /dev/null +++ b/shims/common/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCost.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.execution.adaptive + +import org.apache.spark.sql.errors.QueryExecutionErrors +import org.apache.spark.sql.execution.SparkPlan + +class GlutenCost(val eval: CostEvaluator, val plan: SparkPlan) extends Cost { + override def compare(that: Cost): Int = that match { + case that: GlutenCost if plan eq that.plan => + 0 + case that: GlutenCost if plan == that.plan => + // Plans are identical. Considers the newer one as having lower cost. + -(plan.id - that.plan.id) + case that: GlutenCost => + // Plans are different. Use the delegated cost evaluator. + assert(eval == that.eval) + eval.evaluateCost(plan).compare(eval.evaluateCost(that.plan)) + case _ => + throw QueryExecutionErrors.cannotCompareCostWithTargetCostError(that.toString) + } + + override def hashCode(): Int = throw new UnsupportedOperationException() + + override def equals(obj: Any): Boolean = obj match { + case that: Cost => compare(that) == 0 + case _ => false + } +} diff --git a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala index db8723c710fc..2016352ad86b 100644 --- a/shims/spark32/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala +++ b/shims/spark32/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -16,24 +16,11 @@ */ package org.apache.spark.sql.execution.adaptive -import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution.SparkPlan -case class GlutenCost(value: Long, plan: SparkPlan) extends Cost { - override def compare(that: Cost): Int = that match { - case GlutenCost(thatValue, thatPlan) => - if (value < thatValue || (value == thatValue && !plan.eq(thatPlan))) -1 - else if (value > thatValue) 1 - else 0 - case _ => - throw QueryExecutionErrors.cannotCompareCostWithTargetCostError(that.toString) - } -} - /** This [[CostEvaluator]] is to force use the new physical plan when cost is equal. */ case class GlutenCostEvaluator() extends CostEvaluator { override def evaluateCost(plan: SparkPlan): Cost = { - val simpleCost = SimpleCostEvaluator.evaluateCost(plan).asInstanceOf[SimpleCost] - GlutenCost(simpleCost.value, plan) + new GlutenCost(SimpleCostEvaluator, plan) } } diff --git a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala index 969feb8ac3f8..6b52dd772940 100644 --- a/shims/spark33/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala +++ b/shims/spark33/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -17,28 +17,13 @@ package org.apache.spark.sql.execution.adaptive import org.apache.spark.sql.catalyst.SQLConfHelper -import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.internal.SQLConf -case class GlutenCost(value: Long, plan: SparkPlan) extends Cost { - override def compare(that: Cost): Int = that match { - case GlutenCost(thatValue, thatPlan) => - if (value < thatValue || (value == thatValue && !plan.eq(thatPlan))) -1 - else if (value > thatValue) 1 - else 0 - case _ => - throw QueryExecutionErrors.cannotCompareCostWithTargetCostError(that.toString) - } -} - /** This [[CostEvaluator]] is to force use the new physical plan when cost is equal. */ case class GlutenCostEvaluator() extends CostEvaluator with SQLConfHelper { override def evaluateCost(plan: SparkPlan): Cost = { val forceOptimizeSkewedJoin = conf.getConf(SQLConf.ADAPTIVE_FORCE_OPTIMIZE_SKEWED_JOIN) - val simpleCost = SimpleCostEvaluator(forceOptimizeSkewedJoin) - .evaluateCost(plan) - .asInstanceOf[SimpleCost] - GlutenCost(simpleCost.value, plan) + new GlutenCost(SimpleCostEvaluator(forceOptimizeSkewedJoin), plan) } } diff --git a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala index 969feb8ac3f8..6b52dd772940 100644 --- a/shims/spark34/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala +++ b/shims/spark34/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -17,28 +17,13 @@ package org.apache.spark.sql.execution.adaptive import org.apache.spark.sql.catalyst.SQLConfHelper -import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.internal.SQLConf -case class GlutenCost(value: Long, plan: SparkPlan) extends Cost { - override def compare(that: Cost): Int = that match { - case GlutenCost(thatValue, thatPlan) => - if (value < thatValue || (value == thatValue && !plan.eq(thatPlan))) -1 - else if (value > thatValue) 1 - else 0 - case _ => - throw QueryExecutionErrors.cannotCompareCostWithTargetCostError(that.toString) - } -} - /** This [[CostEvaluator]] is to force use the new physical plan when cost is equal. */ case class GlutenCostEvaluator() extends CostEvaluator with SQLConfHelper { override def evaluateCost(plan: SparkPlan): Cost = { val forceOptimizeSkewedJoin = conf.getConf(SQLConf.ADAPTIVE_FORCE_OPTIMIZE_SKEWED_JOIN) - val simpleCost = SimpleCostEvaluator(forceOptimizeSkewedJoin) - .evaluateCost(plan) - .asInstanceOf[SimpleCost] - GlutenCost(simpleCost.value, plan) + new GlutenCost(SimpleCostEvaluator(forceOptimizeSkewedJoin), plan) } } diff --git a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala index 969feb8ac3f8..6b52dd772940 100644 --- a/shims/spark35/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala +++ b/shims/spark35/src/main/scala/org/apache/spark/sql/execution/adaptive/GlutenCostEvaluator.scala @@ -17,28 +17,13 @@ package org.apache.spark.sql.execution.adaptive import org.apache.spark.sql.catalyst.SQLConfHelper -import org.apache.spark.sql.errors.QueryExecutionErrors import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.internal.SQLConf -case class GlutenCost(value: Long, plan: SparkPlan) extends Cost { - override def compare(that: Cost): Int = that match { - case GlutenCost(thatValue, thatPlan) => - if (value < thatValue || (value == thatValue && !plan.eq(thatPlan))) -1 - else if (value > thatValue) 1 - else 0 - case _ => - throw QueryExecutionErrors.cannotCompareCostWithTargetCostError(that.toString) - } -} - /** This [[CostEvaluator]] is to force use the new physical plan when cost is equal. */ case class GlutenCostEvaluator() extends CostEvaluator with SQLConfHelper { override def evaluateCost(plan: SparkPlan): Cost = { val forceOptimizeSkewedJoin = conf.getConf(SQLConf.ADAPTIVE_FORCE_OPTIMIZE_SKEWED_JOIN) - val simpleCost = SimpleCostEvaluator(forceOptimizeSkewedJoin) - .evaluateCost(plan) - .asInstanceOf[SimpleCost] - GlutenCost(simpleCost.value, plan) + new GlutenCost(SimpleCostEvaluator(forceOptimizeSkewedJoin), plan) } }