diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt index 2debbbd1c1873..b6748ac78a295 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt @@ -1,69 +1,73 @@ == Physical Plan == -AdaptiveSparkPlan (68) +AdaptiveSparkPlan (73) +- == Final Plan == - VeloxColumnarToRowExec (44) - +- ^ ProjectExecTransformer (42) - +- ^ TopNTransformer (41) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (49) + +- ^ ProjectExecTransformer (47) + +- ^ TopNTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ^ InputAdapter (44) + +- ^ ColumnarExchange (43) + +- ^ TopNTransformer (41) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- ^ InputAdapter (9) + : : +- ^ BroadcastQueryStage (8) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- ^ InputAdapter (19) + : +- ^ BroadcastQueryStage (18) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ BroadcastQueryStage (27) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- Project (63) - +- BroadcastHashJoin Inner BuildRight (62) - :- Project (58) - : +- BroadcastHashJoin Inner BuildRight (57) - : :- Project (52) - : : +- BroadcastHashJoin Inner BuildRight (51) - : : :- Filter (46) - : : : +- Scan parquet (45) - : : +- BroadcastExchange (50) - : : +- Project (49) - : : +- Filter (48) - : : +- Scan parquet (47) - : +- BroadcastExchange (56) - : +- Project (55) - : +- Filter (54) - : +- Scan parquet (53) - +- BroadcastExchange (61) - +- Filter (60) - +- Scan parquet (59) + TakeOrderedAndProject (72) + +- HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Project (68) + +- BroadcastHashJoin Inner BuildRight (67) + :- Project (63) + : +- BroadcastHashJoin Inner BuildRight (62) + : :- Project (57) + : : +- BroadcastHashJoin Inner BuildRight (56) + : : :- Filter (51) + : : : +- Scan parquet (50) + : : +- BroadcastExchange (55) + : : +- Project (54) + : : +- Filter (53) + : : +- Scan parquet (52) + : +- BroadcastExchange (61) + : +- Project (60) + : +- Filter (59) + : +- Scan parquet (58) + +- BroadcastExchange (66) + +- Filter (65) + +- Scan parquet (64) (1) Scan parquet @@ -243,130 +247,148 @@ Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], false -(42) ProjectExecTransformer +(42) WholeStageCodegenTransformer (X) +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: false + +(43) ColumnarExchange +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], [plan_id=X], [id=#X] + +(44) InputAdapter +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(45) InputIteratorTransformer +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(46) TopNTransformer +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], true + +(47) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(43) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(44) VeloxColumnarToRowExec +(49) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(45) Scan parquet +(50) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(46) Filter +(51) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(47) Scan parquet +(52) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(48) Filter +(53) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(49) Project +(54) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(50) BroadcastExchange +(55) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(51) BroadcastHashJoin +(56) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(52) Project +(57) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(53) Scan parquet +(58) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(54) Filter +(59) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(55) Project +(60) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(56) BroadcastExchange +(61) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(57) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(59) Scan parquet +(64) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(60) Filter +(65) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(61) BroadcastExchange +(66) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) Project +(68) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(64) HashAggregate +(69) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(65) Exchange +(70) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(71) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) TakeOrderedAndProject +(72) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(68) AdaptiveSparkPlan +(73) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt index 6cdf9e3754f99..b14dd9af62978 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt @@ -1,87 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (86) +- == Final Plan == - VeloxColumnarToRowExec (54) - +- ^ ProjectExecTransformer (52) - +- ^ TopNTransformer (51) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32) - +- ReusedExchange (31) + TakeOrderedAndProject (53) + +- VeloxColumnarToRowExec (52) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ ShuffleQueryStage (15) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + :- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ BroadcastQueryStage (32) + +- ReusedExchange (31) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- BroadcastHashJoin Inner BuildRight (81) - :- Project (69) - : +- BroadcastHashJoin Inner BuildLeft (68) - : :- BroadcastExchange (57) - : : +- Filter (56) - : : +- Scan parquet (55) - : +- BroadcastHashJoin LeftSemi BuildRight (67) - : :- Filter (59) - : : +- Scan parquet (58) - : +- BroadcastExchange (66) - : +- Project (65) - : +- Filter (64) - : +- HashAggregate (63) - : +- Exchange (62) - : +- HashAggregate (61) - : +- Scan parquet (60) - +- BroadcastExchange (80) - +- BroadcastHashJoin LeftSemi BuildRight (79) - :- Filter (71) - : +- Scan parquet (70) - +- BroadcastExchange (78) - +- Project (77) - +- Filter (76) - +- HashAggregate (75) - +- Exchange (74) - +- HashAggregate (73) - +- Scan parquet (72) + TakeOrderedAndProject (85) + +- HashAggregate (84) + +- Exchange (83) + +- HashAggregate (82) + +- Project (81) + +- BroadcastHashJoin Inner BuildRight (80) + :- Project (68) + : +- BroadcastHashJoin Inner BuildLeft (67) + : :- BroadcastExchange (56) + : : +- Filter (55) + : : +- Scan parquet (54) + : +- BroadcastHashJoin LeftSemi BuildRight (66) + : :- Filter (58) + : : +- Scan parquet (57) + : +- BroadcastExchange (65) + : +- Project (64) + : +- Filter (63) + : +- HashAggregate (62) + : +- Exchange (61) + : +- HashAggregate (60) + : +- Scan parquet (59) + +- BroadcastExchange (79) + +- BroadcastHashJoin LeftSemi BuildRight (78) + :- Filter (70) + : +- Scan parquet (69) + +- BroadcastExchange (77) + +- Project (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Scan parquet (71) (1) Scan parquet @@ -298,184 +297,180 @@ Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) TopNTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], false - -(52) ProjectExecTransformer -Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +(51) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: false -(53) WholeStageCodegenTransformer (X) +(52) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: false -(54) VeloxColumnarToRowExec +(53) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(55) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(56) Filter +(55) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(57) BroadcastExchange +(56) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(58) Scan parquet +(57) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(59) Filter +(58) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(60) Scan parquet +(59) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(61) HashAggregate +(60) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(62) Exchange +(61) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(62) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(64) Filter +(63) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(65) Project +(64) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(66) BroadcastExchange +(65) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(68) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(68) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(70) Scan parquet +(69) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(71) Filter +(70) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(72) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(73) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(74) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(76) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(77) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(78) BroadcastExchange +(77) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(80) BroadcastExchange +(79) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(82) Project +(81) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(83) HashAggregate +(82) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(84) Exchange +(83) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(84) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(86) TakeOrderedAndProject +(85) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(87) AdaptiveSparkPlan +(86) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt index adca882b2efe0..2b0a43ea6febc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt @@ -1,92 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ ProjectExecTransformer (58) - +- ^ TopNTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ BroadcastQueryStage (5) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + TakeOrderedAndProject (59) + +- VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ BroadcastQueryStage (5) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- ^ InputAdapter (24) + : : +- ^ BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- BroadcastHashJoin Inner BuildRight (87) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (76) - : : +- BroadcastHashJoin Inner BuildLeft (75) - : : :- BroadcastExchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- BroadcastHashJoin LeftAnti BuildRight (74) - : : :- BroadcastHashJoin LeftSemi BuildRight (69) - : : : :- Project (66) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (68) - : : : +- Scan parquet (67) - : : +- BroadcastExchange (73) - : : +- Project (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- BroadcastExchange (80) - : +- Project (79) - : +- Filter (78) - : +- Scan parquet (77) - +- BroadcastExchange (86) - +- Project (85) - +- Filter (84) - +- Scan parquet (83) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -329,177 +328,173 @@ Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) TopNTransformer -Input [2]: [s_name#X, numwait#X] -Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], false - -(58) ProjectExecTransformer -Output [2]: [s_name#X, numwait#X] +(57) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] +Arguments: false -(59) WholeStageCodegenTransformer (X) +(58) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -Arguments: false -(60) VeloxColumnarToRowExec +(59) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] +Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(61) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(67) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(68) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(70) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(71) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(72) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(73) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(74) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(75) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(76) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(77) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(78) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(79) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(80) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(81) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(82) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(83) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(84) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(85) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(86) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(87) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(89) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(90) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(92) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(93) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt index 49a27cb8273a5..ca16c5508fae7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ ProjectExecTransformer (33) - +- ^ TopNTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ TopNTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ^ InputAdapter (35) + +- ^ ColumnarExchange (34) + +- ^ TopNTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ BroadcastQueryStage (18) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (53) - +- HashAggregate (52) - +- Exchange (51) - +- HashAggregate (50) - +- Project (49) - +- BroadcastHashJoin Inner BuildRight (48) - :- Project (43) - : +- BroadcastHashJoin Inner BuildLeft (42) - : :- BroadcastExchange (39) - : : +- Project (38) - : : +- Filter (37) - : : +- Scan parquet (36) - : +- Filter (41) - : +- Scan parquet (40) - +- BroadcastExchange (47) - +- Project (46) - +- Filter (45) - +- Scan parquet (44) + TakeOrderedAndProject (58) + +- HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- BroadcastHashJoin Inner BuildRight (53) + :- Project (48) + : +- BroadcastHashJoin Inner BuildLeft (47) + : :- BroadcastExchange (44) + : : +- Project (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Filter (46) + : +- Scan parquet (45) + +- BroadcastExchange (52) + +- Project (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -192,106 +196,124 @@ Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((pr Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], false -(33) ProjectExecTransformer +(33) WholeStageCodegenTransformer (X) +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: false + +(34) ColumnarExchange +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] + +(35) InputAdapter +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(36) InputIteratorTransformer +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(37) TopNTransformer +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true + +(38) ProjectExecTransformer Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(34) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(35) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(36) Scan parquet +(41) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(37) Filter +(42) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(38) Project +(43) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(39) BroadcastExchange +(44) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(40) Scan parquet +(45) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(41) Filter +(46) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(42) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(48) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(44) Scan parquet +(49) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(45) Filter +(50) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(46) Project +(51) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(47) BroadcastExchange +(52) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastHashJoin +(53) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(49) Project +(54) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(50) HashAggregate +(55) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(51) Exchange +(56) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) HashAggregate +(57) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(53) TakeOrderedAndProject +(58) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(54) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt index 76ecf7bff5f3d..29d2cd3c2d0c8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt @@ -1,69 +1,73 @@ == Physical Plan == -AdaptiveSparkPlan (68) +AdaptiveSparkPlan (73) +- == Final Plan == - VeloxColumnarToRowExec (44) - +- ^ ProjectExecTransformer (42) - +- ^ TopNTransformer (41) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (49) + +- ^ ProjectExecTransformer (47) + +- ^ TopNTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ^ InputAdapter (44) + +- ^ ColumnarExchange (43) + +- ^ TopNTransformer (41) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- ^ InputAdapter (9) + : : +- ^ BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- ^ InputAdapter (19) + : +- ^ BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- Project (63) - +- BroadcastHashJoin Inner BuildRight (62) - :- Project (58) - : +- BroadcastHashJoin Inner BuildRight (57) - : :- Project (52) - : : +- BroadcastHashJoin Inner BuildRight (51) - : : :- Filter (46) - : : : +- Scan parquet (45) - : : +- BroadcastExchange (50) - : : +- Project (49) - : : +- Filter (48) - : : +- Scan parquet (47) - : +- BroadcastExchange (56) - : +- Project (55) - : +- Filter (54) - : +- Scan parquet (53) - +- BroadcastExchange (61) - +- Filter (60) - +- Scan parquet (59) + TakeOrderedAndProject (72) + +- HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Project (68) + +- BroadcastHashJoin Inner BuildRight (67) + :- Project (63) + : +- BroadcastHashJoin Inner BuildRight (62) + : :- Project (57) + : : +- BroadcastHashJoin Inner BuildRight (56) + : : :- Filter (51) + : : : +- Scan parquet (50) + : : +- BroadcastExchange (55) + : : +- Project (54) + : : +- Filter (53) + : : +- Scan parquet (52) + : +- BroadcastExchange (61) + : +- Project (60) + : +- Filter (59) + : +- Scan parquet (58) + +- BroadcastExchange (66) + +- Filter (65) + +- Scan parquet (64) (1) Scan parquet @@ -243,130 +247,148 @@ Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], false -(42) ProjectExecTransformer +(42) WholeStageCodegenTransformer (X) +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: false + +(43) ColumnarExchange +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], [plan_id=X], [id=#X] + +(44) InputAdapter +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(45) InputIteratorTransformer +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(46) TopNTransformer +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], true + +(47) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(43) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(44) VeloxColumnarToRowExec +(49) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(45) Scan parquet +(50) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(46) Filter +(51) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(47) Scan parquet +(52) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(48) Filter +(53) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(49) Project +(54) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(50) BroadcastExchange +(55) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(51) BroadcastHashJoin +(56) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(52) Project +(57) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(53) Scan parquet +(58) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(54) Filter +(59) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(55) Project +(60) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(56) BroadcastExchange +(61) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(57) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(59) Scan parquet +(64) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(60) Filter +(65) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(61) BroadcastExchange +(66) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) Project +(68) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(64) HashAggregate +(69) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(65) Exchange +(70) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(71) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) TakeOrderedAndProject +(72) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(68) AdaptiveSparkPlan +(73) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt index abee2656acdae..387110df9f4c2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt @@ -1,87 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (86) +- == Final Plan == - VeloxColumnarToRowExec (54) - +- ^ ProjectExecTransformer (52) - +- ^ TopNTransformer (51) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + TakeOrderedAndProject (53) + +- VeloxColumnarToRowExec (52) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23), Statistics(X) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ ShuffleQueryStage (15), Statistics(X) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + :- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ BroadcastQueryStage (32), Statistics(X) + +- ReusedExchange (31) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- BroadcastHashJoin Inner BuildRight (81) - :- Project (69) - : +- BroadcastHashJoin Inner BuildLeft (68) - : :- BroadcastExchange (57) - : : +- Filter (56) - : : +- Scan parquet (55) - : +- BroadcastHashJoin LeftSemi BuildRight (67) - : :- Filter (59) - : : +- Scan parquet (58) - : +- BroadcastExchange (66) - : +- Project (65) - : +- Filter (64) - : +- HashAggregate (63) - : +- Exchange (62) - : +- HashAggregate (61) - : +- Scan parquet (60) - +- BroadcastExchange (80) - +- BroadcastHashJoin LeftSemi BuildRight (79) - :- Filter (71) - : +- Scan parquet (70) - +- BroadcastExchange (78) - +- Project (77) - +- Filter (76) - +- HashAggregate (75) - +- Exchange (74) - +- HashAggregate (73) - +- Scan parquet (72) + TakeOrderedAndProject (85) + +- HashAggregate (84) + +- Exchange (83) + +- HashAggregate (82) + +- Project (81) + +- BroadcastHashJoin Inner BuildRight (80) + :- Project (68) + : +- BroadcastHashJoin Inner BuildLeft (67) + : :- BroadcastExchange (56) + : : +- Filter (55) + : : +- Scan parquet (54) + : +- BroadcastHashJoin LeftSemi BuildRight (66) + : :- Filter (58) + : : +- Scan parquet (57) + : +- BroadcastExchange (65) + : +- Project (64) + : +- Filter (63) + : +- HashAggregate (62) + : +- Exchange (61) + : +- HashAggregate (60) + : +- Scan parquet (59) + +- BroadcastExchange (79) + +- BroadcastHashJoin LeftSemi BuildRight (78) + :- Filter (70) + : +- Scan parquet (69) + +- BroadcastExchange (77) + +- Project (76) + +- Filter (75) + +- HashAggregate (74) + +- Exchange (73) + +- HashAggregate (72) + +- Scan parquet (71) (1) Scan parquet @@ -298,184 +297,180 @@ Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) TopNTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], false - -(52) ProjectExecTransformer -Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +(51) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: false -(53) WholeStageCodegenTransformer (X) +(52) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: false -(54) VeloxColumnarToRowExec +(53) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(55) Scan parquet +(54) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(56) Filter +(55) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(57) BroadcastExchange +(56) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(58) Scan parquet +(57) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(59) Filter +(58) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(60) Scan parquet +(59) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(61) HashAggregate +(60) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(62) Exchange +(61) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(62) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(64) Filter +(63) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(65) Project +(64) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(66) BroadcastExchange +(65) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(68) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(68) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(70) Scan parquet +(69) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(71) Filter +(70) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(72) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(73) HashAggregate +(72) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(74) Exchange +(73) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) HashAggregate +(74) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(76) Filter +(75) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(77) Project +(76) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(78) BroadcastExchange +(77) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(80) BroadcastExchange +(79) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(82) Project +(81) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(83) HashAggregate +(82) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(84) Exchange +(83) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(84) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(86) TakeOrderedAndProject +(85) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(87) AdaptiveSparkPlan +(86) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt index f64c41e24a366..b99911f2b9feb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt @@ -1,56 +1,55 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ ProjectExecTransformer (33) - +- ^ TopNTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + TakeOrderedAndProject (34) + +- VeloxColumnarToRowExec (33) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (53) - +- HashAggregate (52) - +- Exchange (51) - +- HashAggregate (50) - +- Project (49) - +- BroadcastHashJoin Inner BuildRight (48) - :- Project (43) - : +- BroadcastHashJoin Inner BuildLeft (42) - : :- BroadcastExchange (39) - : : +- Project (38) - : : +- Filter (37) - : : +- Scan parquet (36) - : +- Filter (41) - : +- Scan parquet (40) - +- BroadcastExchange (47) - +- Project (46) - +- Filter (45) - +- Scan parquet (44) + TakeOrderedAndProject (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- Project (42) + : +- BroadcastHashJoin Inner BuildLeft (41) + : :- BroadcastExchange (38) + : : +- Project (37) + : : +- Filter (36) + : : +- Scan parquet (35) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -188,110 +187,106 @@ Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow(( Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(32) TopNTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], false - -(33) ProjectExecTransformer -Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +(32) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: false -(34) WholeStageCodegenTransformer (X) +(33) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: false -(35) VeloxColumnarToRowExec +(34) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(36) Scan parquet +(35) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(37) Filter +(36) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(38) Project +(37) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(39) BroadcastExchange +(38) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(40) Scan parquet +(39) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(41) Filter +(40) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(42) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(42) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(44) Scan parquet +(43) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(45) Filter +(44) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(46) Project +(45) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(47) BroadcastExchange +(46) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(49) Project +(48) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(50) HashAggregate +(49) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(51) Exchange +(50) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) HashAggregate +(51) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(53) TakeOrderedAndProject +(52) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(54) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt index 1dfce392f67b8..ab3d73dde692f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt @@ -1,69 +1,68 @@ == Physical Plan == -AdaptiveSparkPlan (68) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (44) - +- ^ ProjectExecTransformer (42) - +- ^ TopNTransformer (41) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + TakeOrderedAndProject (43) + +- VeloxColumnarToRowExec (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- ^ InputAdapter (9) + : : +- ^ BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- ^ InputAdapter (19) + : +- ^ BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (67) - +- HashAggregate (66) - +- Exchange (65) - +- HashAggregate (64) - +- Project (63) - +- BroadcastHashJoin Inner BuildRight (62) - :- Project (58) - : +- BroadcastHashJoin Inner BuildRight (57) - : :- Project (52) - : : +- BroadcastHashJoin Inner BuildRight (51) - : : :- Filter (46) - : : : +- Scan parquet (45) - : : +- BroadcastExchange (50) - : : +- Project (49) - : : +- Filter (48) - : : +- Scan parquet (47) - : +- BroadcastExchange (56) - : +- Project (55) - : +- Filter (54) - : +- Scan parquet (53) - +- BroadcastExchange (61) - +- Filter (60) - +- Scan parquet (59) + TakeOrderedAndProject (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- BroadcastHashJoin Inner BuildRight (61) + :- Project (57) + : +- BroadcastHashJoin Inner BuildRight (56) + : :- Project (51) + : : +- BroadcastHashJoin Inner BuildRight (50) + : : :- Filter (45) + : : : +- Scan parquet (44) + : : +- BroadcastExchange (49) + : : +- Project (48) + : : +- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (55) + : +- Project (54) + : +- Filter (53) + : +- Scan parquet (52) + +- BroadcastExchange (60) + +- Filter (59) + +- Scan parquet (58) (1) Scan parquet @@ -242,137 +241,133 @@ Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(41) TopNTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: X, [revenue#X DESC NULLS LAST], false - -(42) ProjectExecTransformer -Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +(41) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: false -(43) WholeStageCodegenTransformer (X) +(42) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: false -(44) VeloxColumnarToRowExec +(43) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(45) Scan parquet +(44) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(46) Filter +(45) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(47) Scan parquet +(46) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(48) Filter +(47) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(49) Project +(48) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(50) BroadcastExchange +(49) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(51) BroadcastHashJoin +(50) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(52) Project +(51) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(53) Scan parquet +(52) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(54) Filter +(53) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(55) Project +(54) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(56) BroadcastExchange +(55) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(57) BroadcastHashJoin +(56) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(58) Project +(57) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(59) Scan parquet +(58) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(60) Filter +(59) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(61) BroadcastExchange +(60) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) BroadcastHashJoin +(61) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) Project +(62) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(64) HashAggregate +(63) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(65) Exchange +(64) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(66) HashAggregate +(65) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) TakeOrderedAndProject +(66) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(68) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt index b806a7740b581..1dab9dab4bcc6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt @@ -1,87 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (54) - +- ^ ProjectExecTransformer (52) - +- ^ TopNTransformer (51) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (59) + +- ^ ProjectExecTransformer (57) + +- ^ TopNTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ColumnarExchange (53) + +- ^ TopNTransformer (51) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23), Statistics(X) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ ShuffleQueryStage (15), Statistics(X) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + :- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ BroadcastQueryStage (32), Statistics(X) + +- ReusedExchange (31) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- BroadcastHashJoin Inner BuildRight (81) - :- Project (69) - : +- BroadcastHashJoin Inner BuildLeft (68) - : :- BroadcastExchange (57) - : : +- Filter (56) - : : +- Scan parquet (55) - : +- BroadcastHashJoin LeftSemi BuildRight (67) - : :- Filter (59) - : : +- Scan parquet (58) - : +- BroadcastExchange (66) - : +- Project (65) - : +- Filter (64) - : +- HashAggregate (63) - : +- Exchange (62) - : +- HashAggregate (61) - : +- Scan parquet (60) - +- BroadcastExchange (80) - +- BroadcastHashJoin LeftSemi BuildRight (79) - :- Filter (71) - : +- Scan parquet (70) - +- BroadcastExchange (78) - +- Project (77) - +- Filter (76) - +- HashAggregate (75) - +- Exchange (74) - +- HashAggregate (73) - +- Scan parquet (72) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (74) + : +- BroadcastHashJoin Inner BuildLeft (73) + : :- BroadcastExchange (62) + : : +- Filter (61) + : : +- Scan parquet (60) + : +- BroadcastHashJoin LeftSemi BuildRight (72) + : :- Filter (64) + : : +- Scan parquet (63) + : +- BroadcastExchange (71) + : +- Project (70) + : +- Filter (69) + : +- HashAggregate (68) + : +- Exchange (67) + : +- HashAggregate (66) + : +- Scan parquet (65) + +- BroadcastExchange (85) + +- BroadcastHashJoin LeftSemi BuildRight (84) + :- Filter (76) + : +- Scan parquet (75) + +- BroadcastExchange (83) + +- Project (82) + +- Filter (81) + +- HashAggregate (80) + +- Exchange (79) + +- HashAggregate (78) + +- Scan parquet (77) (1) Scan parquet @@ -306,184 +310,202 @@ Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], false -(52) ProjectExecTransformer +(52) WholeStageCodegenTransformer (X) +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: false + +(53) ColumnarExchange +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], [plan_id=X], [id=#X] + +(54) InputAdapter +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(55) InputIteratorTransformer +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(56) TopNTransformer +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true + +(57) ProjectExecTransformer Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(53) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(54) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(55) Scan parquet +(60) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(56) Filter +(61) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(57) BroadcastExchange +(62) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(58) Scan parquet +(63) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(59) Filter +(64) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(60) Scan parquet +(65) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(61) HashAggregate +(66) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(62) Exchange +(67) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(68) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(64) Filter +(69) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(65) Project +(70) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(66) BroadcastExchange +(71) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(68) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(69) Project +(74) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(70) Scan parquet +(75) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(71) Filter +(76) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(72) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(73) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(74) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(76) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(77) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(78) BroadcastExchange +(83) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(80) BroadcastExchange +(85) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(82) Project +(87) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(83) HashAggregate +(88) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(84) Exchange +(89) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(90) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(86) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(87) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt index f4c79f4df4ef2..d0e1db856bb36 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (54) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ ProjectExecTransformer (33) - +- ^ TopNTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ TopNTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ^ InputAdapter (35) + +- ^ ColumnarExchange (34) + +- ^ TopNTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (53) - +- HashAggregate (52) - +- Exchange (51) - +- HashAggregate (50) - +- Project (49) - +- BroadcastHashJoin Inner BuildRight (48) - :- Project (43) - : +- BroadcastHashJoin Inner BuildLeft (42) - : :- BroadcastExchange (39) - : : +- Project (38) - : : +- Filter (37) - : : +- Scan parquet (36) - : +- Filter (41) - : +- Scan parquet (40) - +- BroadcastExchange (47) - +- Project (46) - +- Filter (45) - +- Scan parquet (44) + TakeOrderedAndProject (58) + +- HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- BroadcastHashJoin Inner BuildRight (53) + :- Project (48) + : +- BroadcastHashJoin Inner BuildLeft (47) + : :- BroadcastExchange (44) + : : +- Project (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Filter (46) + : +- Scan parquet (45) + +- BroadcastExchange (52) + +- Project (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -194,108 +198,126 @@ Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice# Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], false -(33) ProjectExecTransformer +(33) WholeStageCodegenTransformer (X) +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: false + +(34) ColumnarExchange +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] + +(35) InputAdapter +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(36) InputIteratorTransformer +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(37) TopNTransformer +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true + +(38) ProjectExecTransformer Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(34) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(35) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(36) Scan parquet +(41) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(37) Filter +(42) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(38) Project +(43) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(39) BroadcastExchange +(44) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(40) Scan parquet +(45) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(41) Filter +(46) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(42) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(43) Project +(48) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(44) Scan parquet +(49) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(45) Filter +(50) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(46) Project +(51) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(47) BroadcastExchange +(52) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastHashJoin +(53) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(49) Project +(54) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(50) HashAggregate +(55) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(51) Exchange +(56) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(52) HashAggregate +(57) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(53) TakeOrderedAndProject +(58) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(54) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt index 1765928eec97e..373e409d549c9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt @@ -1,73 +1,69 @@ == Physical Plan == -AdaptiveSparkPlan (73) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (49) - +- ^ ProjectExecTransformer (47) - +- ^ TopNTransformer (46) - +- ^ InputIteratorTransformer (45) - +- ^ InputAdapter (44) - +- ^ ColumnarExchange (43) - +- ^ TopNTransformer (41) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- ^ ProjectExecTransformer (42) + +- ^ TopNTransformer (41) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- ^ InputAdapter (9) + : : +- ^ BroadcastQueryStage (8) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- ^ InputAdapter (19) + : +- ^ BroadcastQueryStage (18) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ BroadcastQueryStage (27) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (72) - +- HashAggregate (71) - +- Exchange (70) - +- HashAggregate (69) - +- Project (68) - +- BroadcastHashJoin Inner BuildRight (67) - :- Project (63) - : +- BroadcastHashJoin Inner BuildRight (62) - : :- Project (57) - : : +- BroadcastHashJoin Inner BuildRight (56) - : : :- Filter (51) - : : : +- Scan parquet (50) - : : +- BroadcastExchange (55) - : : +- Project (54) - : : +- Filter (53) - : : +- Scan parquet (52) - : +- BroadcastExchange (61) - : +- Project (60) - : +- Filter (59) - : +- Scan parquet (58) - +- BroadcastExchange (66) - +- Filter (65) - +- Scan parquet (64) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -247,148 +243,130 @@ Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], true -(42) WholeStageCodegenTransformer (X) -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: false - -(43) ColumnarExchange -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], [plan_id=X], [id=#X] - -(44) InputAdapter -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(45) InputIteratorTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(46) TopNTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: X, [revenue#X DESC NULLS LAST], true - -(47) ProjectExecTransformer +(42) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(48) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(49) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(50) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(51) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(52) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(53) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(54) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(55) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(57) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(58) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(59) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(60) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(61) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(62) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(63) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(64) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(65) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(66) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(67) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(68) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(69) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(70) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(72) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(73) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt index e85d3438afc5c..f696dd012350f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt @@ -1,91 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (87) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- ^ ProjectExecTransformer (57) - +- ^ TopNTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ColumnarExchange (53) - +- ^ TopNTransformer (51) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32) - +- ReusedExchange (31) + VeloxColumnarToRowExec (54) + +- ^ ProjectExecTransformer (52) + +- ^ TopNTransformer (51) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ ShuffleQueryStage (15) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + :- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ BroadcastQueryStage (32) + +- ReusedExchange (31) +- == Initial Plan == - TakeOrderedAndProject (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (74) - : +- BroadcastHashJoin Inner BuildLeft (73) - : :- BroadcastExchange (62) - : : +- Filter (61) - : : +- Scan parquet (60) - : +- BroadcastHashJoin LeftSemi BuildRight (72) - : :- Filter (64) - : : +- Scan parquet (63) - : +- BroadcastExchange (71) - : +- Project (70) - : +- Filter (69) - : +- HashAggregate (68) - : +- Exchange (67) - : +- HashAggregate (66) - : +- Scan parquet (65) - +- BroadcastExchange (85) - +- BroadcastHashJoin LeftSemi BuildRight (84) - :- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (83) - +- Project (82) - +- Filter (81) - +- HashAggregate (80) - +- Exchange (79) - +- HashAggregate (78) - +- Scan parquet (77) + TakeOrderedAndProject (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- BroadcastHashJoin Inner BuildRight (81) + :- Project (69) + : +- BroadcastHashJoin Inner BuildLeft (68) + : :- BroadcastExchange (57) + : : +- Filter (56) + : : +- Scan parquet (55) + : +- BroadcastHashJoin LeftSemi BuildRight (67) + : :- Filter (59) + : : +- Scan parquet (58) + : +- BroadcastExchange (66) + : +- Project (65) + : +- Filter (64) + : +- HashAggregate (63) + : +- Exchange (62) + : +- HashAggregate (61) + : +- Scan parquet (60) + +- BroadcastExchange (80) + +- BroadcastHashJoin LeftSemi BuildRight (79) + :- Filter (71) + : +- Scan parquet (70) + +- BroadcastExchange (78) + +- Project (77) + +- Filter (76) + +- HashAggregate (75) + +- Exchange (74) + +- HashAggregate (73) + +- Scan parquet (72) (1) Scan parquet @@ -306,198 +302,180 @@ Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true -(52) WholeStageCodegenTransformer (X) -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: false - -(53) ColumnarExchange -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], [plan_id=X], [id=#X] - -(54) InputAdapter -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(55) InputIteratorTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(56) TopNTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true - -(57) ProjectExecTransformer +(52) ProjectExecTransformer Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(58) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(59) VeloxColumnarToRowExec +(54) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(60) Scan parquet +(55) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(61) Filter +(56) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(62) BroadcastExchange +(57) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(58) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(64) Filter +(59) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(65) Scan parquet +(60) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) HashAggregate +(61) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(67) Exchange +(62) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(63) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(69) Filter +(64) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(70) Project +(65) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(71) BroadcastExchange +(66) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(73) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(74) Project +(69) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(75) Scan parquet +(70) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(76) Filter +(71) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(77) Scan parquet +(72) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(78) HashAggregate +(73) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(79) Exchange +(74) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) HashAggregate +(75) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(81) Filter +(76) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(82) Project +(77) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(83) BroadcastExchange +(78) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(84) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(85) BroadcastExchange +(80) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(87) Project +(82) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(88) HashAggregate +(83) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(89) Exchange +(84) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(85) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(91) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(92) AdaptiveSparkPlan +(87) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt index fee654505aca8..bfa6e1174329d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt @@ -1,96 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ ProjectExecTransformer (63) - +- ^ TopNTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ^ InputAdapter (60) - +- ^ ColumnarExchange (59) - +- ^ TopNTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ BroadcastQueryStage (5) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (60) + +- ^ ProjectExecTransformer (58) + +- ^ TopNTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ BroadcastQueryStage (5) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- ^ InputAdapter (24) + : : +- ^ BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (97) - +- HashAggregate (96) - +- Exchange (95) - +- HashAggregate (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (81) - : : +- BroadcastHashJoin Inner BuildLeft (80) - : : :- BroadcastExchange (68) - : : : +- Filter (67) - : : : +- Scan parquet (66) - : : +- BroadcastHashJoin LeftAnti BuildRight (79) - : : :- BroadcastHashJoin LeftSemi BuildRight (74) - : : : :- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (73) - : : : +- Scan parquet (72) - : : +- BroadcastExchange (78) - : : +- Project (77) - : : +- Filter (76) - : : +- Scan parquet (75) - : +- BroadcastExchange (85) - : +- Project (84) - : +- Filter (83) - : +- Scan parquet (82) - +- BroadcastExchange (91) - +- Project (90) - +- Filter (89) - +- Scan parquet (88) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (76) + : : +- BroadcastHashJoin Inner BuildLeft (75) + : : :- BroadcastExchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- BroadcastHashJoin LeftAnti BuildRight (74) + : : :- BroadcastHashJoin LeftSemi BuildRight (69) + : : : :- Project (66) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (68) + : : : +- Scan parquet (67) + : : +- BroadcastExchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- BroadcastExchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- BroadcastExchange (86) + +- Project (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -337,191 +333,173 @@ Results [2]: [s_name#X, count(1)#X AS numwait#X] Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], true -(58) WholeStageCodegenTransformer (X) -Input [2]: [s_name#X, numwait#X] -Arguments: false - -(59) ColumnarExchange -Input [2]: [s_name#X, numwait#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [s_name#X, numwait#X], [plan_id=X], [id=#X] - -(60) InputAdapter -Input [2]: [s_name#X, numwait#X] - -(61) InputIteratorTransformer -Input [2]: [s_name#X, numwait#X] - -(62) TopNTransformer -Input [2]: [s_name#X, numwait#X] -Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], true - -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [2]: [s_name#X, numwait#X] Input [2]: [s_name#X, numwait#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(65) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(66) Scan parquet +(61) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(67) Filter +(62) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(68) BroadcastExchange +(63) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(64) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(65) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) Project +(66) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(72) Scan parquet +(67) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(73) BroadcastExchange +(68) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(74) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(75) Scan parquet +(70) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(76) Filter +(71) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(77) Project +(72) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(78) BroadcastExchange +(73) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(80) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(81) Project +(76) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(82) Scan parquet +(77) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(83) Filter +(78) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(84) Project +(79) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(85) BroadcastExchange +(80) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(82) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(88) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(90) Project +(85) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(91) BroadcastExchange +(86) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(92) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) Project +(88) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(94) HashAggregate +(89) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(95) Exchange +(90) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) HashAggregate +(91) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(97) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(98) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt index 1aa99bb010a3a..4a6559acef01c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt @@ -1,60 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ ProjectExecTransformer (38) - +- ^ TopNTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ^ InputAdapter (35) - +- ^ ColumnarExchange (34) - +- ^ TopNTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- ^ ProjectExecTransformer (33) + +- ^ TopNTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ BroadcastQueryStage (18) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- Exchange (56) - +- HashAggregate (55) - +- Project (54) - +- BroadcastHashJoin Inner BuildRight (53) - :- Project (48) - : +- BroadcastHashJoin Inner BuildLeft (47) - : :- BroadcastExchange (44) - : : +- Project (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Filter (46) - : +- Scan parquet (45) - +- BroadcastExchange (52) - +- Project (51) - +- Filter (50) - +- Scan parquet (49) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -196,124 +192,106 @@ Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((pr Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true -(33) WholeStageCodegenTransformer (X) -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: false - -(34) ColumnarExchange -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] - -(35) InputAdapter -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(36) InputIteratorTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(37) TopNTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true - -(38) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(39) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(40) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(41) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(42) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(43) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(44) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(45) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(46) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(47) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(50) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(51) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(52) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(53) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(54) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(55) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(56) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt index 80b88be476d23..7b90ff535527c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt @@ -1,73 +1,69 @@ == Physical Plan == -AdaptiveSparkPlan (73) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (49) - +- ^ ProjectExecTransformer (47) - +- ^ TopNTransformer (46) - +- ^ InputIteratorTransformer (45) - +- ^ InputAdapter (44) - +- ^ ColumnarExchange (43) - +- ^ TopNTransformer (41) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- ^ ProjectExecTransformer (42) + +- ^ TopNTransformer (41) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- ^ InputAdapter (9) + : : +- ^ BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- ^ InputAdapter (19) + : +- ^ BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (72) - +- HashAggregate (71) - +- Exchange (70) - +- HashAggregate (69) - +- Project (68) - +- BroadcastHashJoin Inner BuildRight (67) - :- Project (63) - : +- BroadcastHashJoin Inner BuildRight (62) - : :- Project (57) - : : +- BroadcastHashJoin Inner BuildRight (56) - : : :- Filter (51) - : : : +- Scan parquet (50) - : : +- BroadcastExchange (55) - : : +- Project (54) - : : +- Filter (53) - : : +- Scan parquet (52) - : +- BroadcastExchange (61) - : +- Project (60) - : +- Filter (59) - : +- Scan parquet (58) - +- BroadcastExchange (66) - +- Filter (65) - +- Scan parquet (64) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -247,148 +243,130 @@ Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], true -(42) WholeStageCodegenTransformer (X) -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: false - -(43) ColumnarExchange -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], [plan_id=X], [id=#X] - -(44) InputAdapter -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(45) InputIteratorTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(46) TopNTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: X, [revenue#X DESC NULLS LAST], true - -(47) ProjectExecTransformer +(42) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(48) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(49) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(50) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(51) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(52) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(53) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(54) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(55) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(57) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(58) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(59) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(60) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(61) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(62) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(63) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(64) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(65) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(66) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(67) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(68) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(69) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(70) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(72) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(73) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt index bfaed83e0fb2e..3536c955cbf77 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt @@ -1,91 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (87) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- ^ ProjectExecTransformer (57) - +- ^ TopNTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ColumnarExchange (53) - +- ^ TopNTransformer (51) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (54) + +- ^ ProjectExecTransformer (52) + +- ^ TopNTransformer (51) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23), Statistics(X) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ ShuffleQueryStage (15), Statistics(X) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + :- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ BroadcastQueryStage (32), Statistics(X) + +- ReusedExchange (31) +- == Initial Plan == - TakeOrderedAndProject (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (74) - : +- BroadcastHashJoin Inner BuildLeft (73) - : :- BroadcastExchange (62) - : : +- Filter (61) - : : +- Scan parquet (60) - : +- BroadcastHashJoin LeftSemi BuildRight (72) - : :- Filter (64) - : : +- Scan parquet (63) - : +- BroadcastExchange (71) - : +- Project (70) - : +- Filter (69) - : +- HashAggregate (68) - : +- Exchange (67) - : +- HashAggregate (66) - : +- Scan parquet (65) - +- BroadcastExchange (85) - +- BroadcastHashJoin LeftSemi BuildRight (84) - :- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (83) - +- Project (82) - +- Filter (81) - +- HashAggregate (80) - +- Exchange (79) - +- HashAggregate (78) - +- Scan parquet (77) + TakeOrderedAndProject (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- BroadcastHashJoin Inner BuildRight (81) + :- Project (69) + : +- BroadcastHashJoin Inner BuildLeft (68) + : :- BroadcastExchange (57) + : : +- Filter (56) + : : +- Scan parquet (55) + : +- BroadcastHashJoin LeftSemi BuildRight (67) + : :- Filter (59) + : : +- Scan parquet (58) + : +- BroadcastExchange (66) + : +- Project (65) + : +- Filter (64) + : +- HashAggregate (63) + : +- Exchange (62) + : +- HashAggregate (61) + : +- Scan parquet (60) + +- BroadcastExchange (80) + +- BroadcastHashJoin LeftSemi BuildRight (79) + :- Filter (71) + : +- Scan parquet (70) + +- BroadcastExchange (78) + +- Project (77) + +- Filter (76) + +- HashAggregate (75) + +- Exchange (74) + +- HashAggregate (73) + +- Scan parquet (72) (1) Scan parquet @@ -306,198 +302,180 @@ Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true -(52) WholeStageCodegenTransformer (X) -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: false - -(53) ColumnarExchange -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], [plan_id=X], [id=#X] - -(54) InputAdapter -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(55) InputIteratorTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(56) TopNTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true - -(57) ProjectExecTransformer +(52) ProjectExecTransformer Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(58) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(59) VeloxColumnarToRowExec +(54) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(60) Scan parquet +(55) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(61) Filter +(56) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(62) BroadcastExchange +(57) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(58) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(64) Filter +(59) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(65) Scan parquet +(60) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) HashAggregate +(61) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(67) Exchange +(62) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(63) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(69) Filter +(64) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(70) Project +(65) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(71) BroadcastExchange +(66) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(73) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(74) Project +(69) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(75) Scan parquet +(70) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(76) Filter +(71) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(77) Scan parquet +(72) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(78) HashAggregate +(73) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(79) Exchange +(74) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) HashAggregate +(75) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(81) Filter +(76) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(82) Project +(77) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(83) BroadcastExchange +(78) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(84) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(85) BroadcastExchange +(80) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(87) Project +(82) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(88) HashAggregate +(83) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(89) Exchange +(84) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(85) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(91) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(92) AdaptiveSparkPlan +(87) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt index 707c47c2dc9ca..d3dc0c818ec81 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt @@ -1,60 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ ProjectExecTransformer (38) - +- ^ TopNTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ^ InputAdapter (35) - +- ^ ColumnarExchange (34) - +- ^ TopNTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- ^ ProjectExecTransformer (33) + +- ^ TopNTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- Exchange (56) - +- HashAggregate (55) - +- Project (54) - +- BroadcastHashJoin Inner BuildRight (53) - :- Project (48) - : +- BroadcastHashJoin Inner BuildLeft (47) - : :- BroadcastExchange (44) - : : +- Project (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Filter (46) - : +- Scan parquet (45) - +- BroadcastExchange (52) - +- Project (51) - +- Filter (50) - +- Scan parquet (49) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -196,124 +192,106 @@ Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((pr Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true -(33) WholeStageCodegenTransformer (X) -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: false - -(34) ColumnarExchange -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] - -(35) InputAdapter -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(36) InputIteratorTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(37) TopNTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true - -(38) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(39) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(40) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(41) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(42) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(43) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(44) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(45) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(46) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(47) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(50) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(51) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(52) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(53) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(54) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(55) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(56) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt index bee44964695a5..3e98d75f24bc8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt @@ -1,73 +1,69 @@ == Physical Plan == -AdaptiveSparkPlan (73) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (49) - +- ^ ProjectExecTransformer (47) - +- ^ TopNTransformer (46) - +- ^ InputIteratorTransformer (45) - +- ^ InputAdapter (44) - +- ^ ColumnarExchange (43) - +- ^ TopNTransformer (41) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- ^ ProjectExecTransformer (42) + +- ^ TopNTransformer (41) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ ProjectExecTransformer (33) + +- ^ FlushableHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- ^ InputAdapter (9) + : : +- ^ BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- ^ InputAdapter (19) + : +- ^ BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (72) - +- HashAggregate (71) - +- Exchange (70) - +- HashAggregate (69) - +- Project (68) - +- BroadcastHashJoin Inner BuildRight (67) - :- Project (63) - : +- BroadcastHashJoin Inner BuildRight (62) - : :- Project (57) - : : +- BroadcastHashJoin Inner BuildRight (56) - : : :- Filter (51) - : : : +- Scan parquet (50) - : : +- BroadcastExchange (55) - : : +- Project (54) - : : +- Filter (53) - : : +- Scan parquet (52) - : +- BroadcastExchange (61) - : +- Project (60) - : +- Filter (59) - : +- Scan parquet (58) - +- BroadcastExchange (66) - +- Filter (65) - +- Scan parquet (64) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -250,151 +246,133 @@ Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], true -(42) WholeStageCodegenTransformer (X) -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: false - -(43) ColumnarExchange -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], [plan_id=X], [id=#X] - -(44) InputAdapter -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(45) InputIteratorTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(46) TopNTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: X, [revenue#X DESC NULLS LAST], true - -(47) ProjectExecTransformer +(42) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(48) WholeStageCodegenTransformer (X) +(43) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(49) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(50) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(51) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(52) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(53) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(54) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(55) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(57) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(58) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(59) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(60) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(61) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(62) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(63) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(64) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(65) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(66) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(67) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(68) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(69) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(70) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(72) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(73) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt index 04b293a034ef0..93d7bdc4e3c33 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt @@ -1,91 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (87) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- ^ ProjectExecTransformer (57) - +- ^ TopNTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ColumnarExchange (53) - +- ^ TopNTransformer (51) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (54) + +- ^ ProjectExecTransformer (52) + +- ^ TopNTransformer (51) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (28) + : +- ^ BroadcastHashJoinExecTransformer Inner (27) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (25) + : +- ^ InputAdapter (24) + : +- ^ BroadcastQueryStage (23), Statistics(X) + : +- ColumnarBroadcastExchange (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ FilterExecTransformer (19) + : +- ^ RegularHashAggregateExecTransformer (18) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ ShuffleQueryStage (15), Statistics(X) + : +- ColumnarExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FlushableHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) + :- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ BroadcastQueryStage (32), Statistics(X) + +- ReusedExchange (31) +- == Initial Plan == - TakeOrderedAndProject (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (74) - : +- BroadcastHashJoin Inner BuildLeft (73) - : :- BroadcastExchange (62) - : : +- Filter (61) - : : +- Scan parquet (60) - : +- BroadcastHashJoin LeftSemi BuildRight (72) - : :- Filter (64) - : : +- Scan parquet (63) - : +- BroadcastExchange (71) - : +- Project (70) - : +- Filter (69) - : +- HashAggregate (68) - : +- Exchange (67) - : +- HashAggregate (66) - : +- Scan parquet (65) - +- BroadcastExchange (85) - +- BroadcastHashJoin LeftSemi BuildRight (84) - :- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (83) - +- Project (82) - +- Filter (81) - +- HashAggregate (80) - +- Exchange (79) - +- HashAggregate (78) - +- Scan parquet (77) + TakeOrderedAndProject (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- BroadcastHashJoin Inner BuildRight (81) + :- Project (69) + : +- BroadcastHashJoin Inner BuildLeft (68) + : :- BroadcastExchange (57) + : : +- Filter (56) + : : +- Scan parquet (55) + : +- BroadcastHashJoin LeftSemi BuildRight (67) + : :- Filter (59) + : : +- Scan parquet (58) + : +- BroadcastExchange (66) + : +- Project (65) + : +- Filter (64) + : +- HashAggregate (63) + : +- Exchange (62) + : +- HashAggregate (61) + : +- Scan parquet (60) + +- BroadcastExchange (80) + +- BroadcastHashJoin LeftSemi BuildRight (79) + :- Filter (71) + : +- Scan parquet (70) + +- BroadcastExchange (78) + +- Project (77) + +- Filter (76) + +- HashAggregate (75) + +- Exchange (74) + +- HashAggregate (73) + +- Scan parquet (72) (1) Scan parquet @@ -310,202 +306,184 @@ Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true -(52) WholeStageCodegenTransformer (X) -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: false - -(53) ColumnarExchange -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], [plan_id=X], [id=#X] - -(54) InputAdapter -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(55) InputIteratorTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(56) TopNTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true - -(57) ProjectExecTransformer +(52) ProjectExecTransformer Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(58) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(59) VeloxColumnarToRowExec +(54) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(60) Scan parquet +(55) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(61) Filter +(56) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(62) BroadcastExchange +(57) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(58) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(64) Filter +(59) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(65) Scan parquet +(60) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) HashAggregate +(61) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(67) Exchange +(62) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(63) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(69) Filter +(64) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(70) Project +(65) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(71) BroadcastExchange +(66) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(73) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(74) Project +(69) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(75) Scan parquet +(70) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(76) Filter +(71) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(77) Scan parquet +(72) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(78) HashAggregate +(73) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(79) Exchange +(74) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) HashAggregate +(75) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(81) Filter +(76) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(82) Project +(77) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(83) BroadcastExchange +(78) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(84) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(85) BroadcastExchange +(80) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(87) Project +(82) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(88) HashAggregate +(83) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(89) Exchange +(84) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(85) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(91) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(92) AdaptiveSparkPlan +(87) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt index 2ce9a9f035b8c..801a9260f06ba 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt @@ -1,60 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ ProjectExecTransformer (38) - +- ^ TopNTransformer (37) - +- ^ InputIteratorTransformer (36) - +- ^ InputAdapter (35) - +- ^ ColumnarExchange (34) - +- ^ TopNTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- ^ ProjectExecTransformer (33) + +- ^ TopNTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner (11) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- ^ InputAdapter (19) + +- ^ BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- Exchange (56) - +- HashAggregate (55) - +- Project (54) - +- BroadcastHashJoin Inner BuildRight (53) - :- Project (48) - : +- BroadcastHashJoin Inner BuildLeft (47) - : :- BroadcastExchange (44) - : : +- Project (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Filter (46) - : +- Scan parquet (45) - +- BroadcastExchange (52) - +- Project (51) - +- Filter (50) - +- Scan parquet (49) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -198,126 +194,108 @@ Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice# Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true -(33) WholeStageCodegenTransformer (X) -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: false - -(34) ColumnarExchange -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] - -(35) InputAdapter -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(36) InputIteratorTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(37) TopNTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true - -(38) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(39) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(40) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(41) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(42) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(43) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(44) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(45) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(46) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(47) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(48) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(50) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(51) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(52) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(53) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(54) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(55) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(56) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt index 889a46147ebbd..7fef87bd3ba2f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt @@ -1,86 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (88) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ ProjectExecTransformer (59) - +- ^ TopNTransformer (58) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- ^ ProjectExecTransformer (64) + +- ^ TopNTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ^ InputAdapter (61) + +- ^ ColumnarExchange (60) + +- ^ TopNTransformer (58) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner (47) + :- ^ InputIteratorTransformer (38) + : +- ^ InputAdapter (37) + : +- ^ ShuffleQueryStage (36) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : :- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ ShuffleQueryStage (21) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ ShuffleQueryStage (29) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (87) - +- HashAggregate (86) - +- Exchange (85) - +- HashAggregate (84) - +- Project (83) - +- ShuffledHashJoin Inner BuildRight (82) - :- Exchange (78) - : +- Project (77) - : +- ShuffledHashJoin Inner BuildRight (76) - : :- Exchange (71) - : : +- Project (70) - : : +- ShuffledHashJoin Inner BuildRight (69) - : : :- Exchange (64) - : : : +- Filter (63) - : : : +- Scan parquet (62) - : : +- Exchange (68) - : : +- Project (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (75) - : +- Project (74) - : +- Filter (73) - : +- Scan parquet (72) - +- Exchange (81) - +- Filter (80) - +- Scan parquet (79) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -322,142 +326,160 @@ Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], false -(59) ProjectExecTransformer +(59) WholeStageCodegenTransformer (X) +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: false + +(60) ColumnarExchange +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], [plan_id=X], [id=#X] + +(61) InputAdapter +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(62) InputIteratorTransformer +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(63) TopNTransformer +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], true + +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(60) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(61) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(62) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(63) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(64) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(67) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(68) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(70) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(71) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(72) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(73) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(74) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(75) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(77) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(78) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(80) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(81) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(83) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(84) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(85) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(88) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt index b08b33f900ee2..1dcadd7d1812e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt @@ -1,97 +1,101 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ ProjectExecTransformer (63) - +- ^ TopNTransformer (62) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) - :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) - :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51) - +- ReusedExchange (50) + VeloxColumnarToRowExec (70) + +- ^ ProjectExecTransformer (68) + +- ^ TopNTransformer (67) + +- ^ InputIteratorTransformer (66) + +- ^ InputAdapter (65) + +- ^ ColumnarExchange (64) + +- ^ TopNTransformer (62) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner (58) + :- ^ InputIteratorTransformer (41) + : +- ^ InputAdapter (40) + : +- ^ ShuffleQueryStage (39) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ^ InputAdapter (33) + : +- ^ ShuffleQueryStage (32) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : :- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ^ InputAdapter (23) + : +- ^ ShuffleQueryStage (22) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + :- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ ShuffleQueryStage (47) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (97) - +- HashAggregate (96) - +- HashAggregate (95) - +- Project (94) - +- ShuffledHashJoin Inner BuildRight (93) - :- Exchange (82) - : +- Project (81) - : +- ShuffledHashJoin Inner BuildLeft (80) - : :- Exchange (68) - : : +- Filter (67) - : : +- Scan parquet (66) - : +- Exchange (79) - : +- ShuffledHashJoin LeftSemi BuildRight (78) - : :- Exchange (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- Project (77) - : +- Filter (76) - : +- HashAggregate (75) - : +- Exchange (74) - : +- HashAggregate (73) - : +- Scan parquet (72) - +- ShuffledHashJoin LeftSemi BuildRight (92) - :- Exchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- Project (91) - +- Filter (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Scan parquet (86) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -357,180 +361,198 @@ Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], false -(63) ProjectExecTransformer +(63) WholeStageCodegenTransformer (X) +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: false + +(64) ColumnarExchange +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], [plan_id=X], [id=#X] + +(65) InputAdapter +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(66) InputIteratorTransformer +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(67) TopNTransformer +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true + +(68) ProjectExecTransformer Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(64) WholeStageCodegenTransformer (X) +(69) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(65) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(66) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(67) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(68) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(71) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(72) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(73) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(74) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(76) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(77) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(78) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(81) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(82) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(83) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(84) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(85) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(87) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(88) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(90) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(91) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(94) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(95) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(96) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(97) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(98) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt index 4375f571b0bc3..a7c94c61af845 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt @@ -1,115 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (120) +AdaptiveSparkPlan (125) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- ^ ProjectExecTransformer (81) - +- ^ TopNTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner (70) - :- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) - : :- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ^ InputAdapter (22) - : : : +- ^ ShuffleQueryStage (21) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ^ InputAdapter (31) - : : +- ^ ShuffleQueryStage (30) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ^ InputAdapter (68) - +- ^ ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (88) + +- ^ ProjectExecTransformer (86) + +- ^ TopNTransformer (85) + +- ^ InputIteratorTransformer (84) + +- ^ InputAdapter (83) + +- ^ ColumnarExchange (82) + +- ^ TopNTransformer (80) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner (70) + :- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ^ InputAdapter (22) + : : : +- ^ ShuffleQueryStage (21) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ^ InputAdapter (31) + : : +- ^ ShuffleQueryStage (30) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ NoopFilter (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ NoopFilter (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ^ InputAdapter (68) + +- ^ ShuffleQueryStage (67) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ NoopFilter (63) + +- ^ Scan parquet (62) +- == Initial Plan == - TakeOrderedAndProject (119) - +- HashAggregate (118) - +- Exchange (117) - +- HashAggregate (116) - +- Project (115) - +- ShuffledHashJoin Inner BuildRight (114) - :- Exchange (109) - : +- Project (108) - : +- ShuffledHashJoin Inner BuildRight (107) - : :- Exchange (102) - : : +- Project (101) - : : +- ShuffledHashJoin Inner BuildLeft (100) - : : :- Exchange (86) - : : : +- Filter (85) - : : : +- Scan parquet (84) - : : +- Exchange (99) - : : +- ShuffledHashJoin LeftAnti BuildRight (98) - : : :- ShuffledHashJoin LeftSemi BuildRight (93) - : : : :- Exchange (90) - : : : : +- Project (89) - : : : : +- Filter (88) - : : : : +- Scan parquet (87) - : : : +- Exchange (92) - : : : +- Scan parquet (91) - : : +- Exchange (97) - : : +- Project (96) - : : +- Filter (95) - : : +- Scan parquet (94) - : +- Exchange (106) - : +- Project (105) - : +- Filter (104) - : +- Scan parquet (103) - +- Exchange (113) - +- Project (112) - +- Filter (111) - +- Scan parquet (110) + TakeOrderedAndProject (124) + +- HashAggregate (123) + +- Exchange (122) + +- HashAggregate (121) + +- Project (120) + +- ShuffledHashJoin Inner BuildRight (119) + :- Exchange (114) + : +- Project (113) + : +- ShuffledHashJoin Inner BuildRight (112) + : :- Exchange (107) + : : +- Project (106) + : : +- ShuffledHashJoin Inner BuildLeft (105) + : : :- Exchange (91) + : : : +- Filter (90) + : : : +- Scan parquet (89) + : : +- Exchange (104) + : : +- ShuffledHashJoin LeftAnti BuildRight (103) + : : :- ShuffledHashJoin LeftSemi BuildRight (98) + : : : :- Exchange (95) + : : : : +- Project (94) + : : : : +- Filter (93) + : : : : +- Scan parquet (92) + : : : +- Exchange (97) + : : : +- Scan parquet (96) + : : +- Exchange (102) + : : +- Project (101) + : : +- Filter (100) + : : +- Scan parquet (99) + : +- Exchange (111) + : +- Project (110) + : +- Filter (109) + : +- Scan parquet (108) + +- Exchange (118) + +- Project (117) + +- Filter (116) + +- Scan parquet (115) (1) Scan parquet @@ -440,189 +444,207 @@ Results [2]: [s_name#X, count(1)#X AS numwait#X] Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], false -(81) ProjectExecTransformer +(81) WholeStageCodegenTransformer (X) +Input [2]: [s_name#X, numwait#X] +Arguments: false + +(82) ColumnarExchange +Input [2]: [s_name#X, numwait#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [s_name#X, numwait#X], [plan_id=X], [id=#X] + +(83) InputAdapter +Input [2]: [s_name#X, numwait#X] + +(84) InputIteratorTransformer +Input [2]: [s_name#X, numwait#X] + +(85) TopNTransformer +Input [2]: [s_name#X, numwait#X] +Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], true + +(86) ProjectExecTransformer Output [2]: [s_name#X, numwait#X] Input [2]: [s_name#X, numwait#X] -(82) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(83) VeloxColumnarToRowExec +(88) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(84) Scan parquet +(89) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(90) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(86) Exchange +(91) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(92) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(88) Filter +(93) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(89) Project +(94) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(90) Exchange +(95) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(96) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(92) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(94) Scan parquet +(99) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(95) Filter +(100) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(96) Project +(101) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(97) Exchange +(102) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(103) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(99) Exchange +(104) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(101) Project +(106) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(102) Exchange +(107) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(108) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(104) Filter +(109) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(105) Project +(110) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(106) Exchange +(111) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(108) Project +(113) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(109) Exchange +(114) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(110) Scan parquet +(115) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(111) Filter +(116) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(112) Project +(117) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(113) Exchange +(118) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(115) Project +(120) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(116) HashAggregate +(121) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(117) Exchange +(122) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) HashAggregate +(123) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(119) TakeOrderedAndProject +(124) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(120) AdaptiveSparkPlan +(125) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt index 5bab0b6225db0..8927e80a6cd15 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt @@ -1,61 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (60) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ ProjectExecTransformer (38) - +- ^ TopNTransformer (37) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + TakeOrderedAndProject (39) + +- VeloxColumnarToRowExec (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (59) - +- HashAggregate (58) - +- HashAggregate (57) - +- Project (56) - +- ShuffledHashJoin Inner BuildRight (55) - :- Exchange (50) - : +- Project (49) - : +- ShuffledHashJoin Inner BuildLeft (48) - : :- Exchange (44) - : : +- Project (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Exchange (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + TakeOrderedAndProject (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (49) + : +- Project (48) + : +- ShuffledHashJoin Inner BuildLeft (47) + : :- Exchange (43) + : : +- Project (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Exchange (53) + +- Project (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -211,114 +210,110 @@ Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow(( Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(37) TopNTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], false - -(38) ProjectExecTransformer -Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +(37) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: false -(39) WholeStageCodegenTransformer (X) +(38) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: false -(40) VeloxColumnarToRowExec +(39) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(41) Scan parquet +(40) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(42) Filter +(41) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(43) Project +(42) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(44) Exchange +(43) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Scan parquet +(44) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(46) Filter +(45) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(47) Exchange +(46) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(49) Project +(48) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(50) Exchange +(49) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(50) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(52) Filter +(51) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(53) Project +(52) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(54) Exchange +(53) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(56) Project +(55) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(57) HashAggregate +(56) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(58) HashAggregate +(57) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(59) TakeOrderedAndProject +(58) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(60) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt index f75151af57b7b..2b450c890975e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt @@ -1,86 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (88) +AdaptiveSparkPlan (87) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ ProjectExecTransformer (59) - +- ^ TopNTransformer (58) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + TakeOrderedAndProject (60) + +- VeloxColumnarToRowExec (59) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner (47) + :- ^ InputIteratorTransformer (38) + : +- ^ InputAdapter (37) + : +- ^ ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : :- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (87) - +- HashAggregate (86) - +- Exchange (85) - +- HashAggregate (84) - +- Project (83) - +- ShuffledHashJoin Inner BuildRight (82) - :- Exchange (78) - : +- Project (77) - : +- ShuffledHashJoin Inner BuildRight (76) - : :- Exchange (71) - : : +- Project (70) - : : +- ShuffledHashJoin Inner BuildRight (69) - : : :- Exchange (64) - : : : +- Filter (63) - : : : +- Scan parquet (62) - : : +- Exchange (68) - : : +- Project (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (75) - : +- Project (74) - : +- Filter (73) - : +- Scan parquet (72) - +- Exchange (81) - +- Filter (80) - +- Scan parquet (79) + TakeOrderedAndProject (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- ShuffledHashJoin Inner BuildRight (81) + :- Exchange (77) + : +- Project (76) + : +- ShuffledHashJoin Inner BuildRight (75) + : :- Exchange (70) + : : +- Project (69) + : : +- ShuffledHashJoin Inner BuildRight (68) + : : :- Exchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- Exchange (67) + : : +- Project (66) + : : +- Filter (65) + : : +- Scan parquet (64) + : +- Exchange (74) + : +- Project (73) + : +- Filter (72) + : +- Scan parquet (71) + +- Exchange (80) + +- Filter (79) + +- Scan parquet (78) (1) Scan parquet @@ -318,146 +317,142 @@ Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(58) TopNTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: X, [revenue#X DESC NULLS LAST], false - -(59) ProjectExecTransformer -Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +(58) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: false -(60) WholeStageCodegenTransformer (X) +(59) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: false -(61) VeloxColumnarToRowExec +(60) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(62) Scan parquet +(61) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(63) Filter +(62) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(64) Exchange +(63) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(64) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(66) Filter +(65) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(67) Project +(66) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(68) Exchange +(67) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) ShuffledHashJoin +(68) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(70) Project +(69) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(71) Exchange +(70) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(72) Scan parquet +(71) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(73) Filter +(72) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(74) Project +(73) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(75) Exchange +(74) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) ShuffledHashJoin +(75) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(77) Project +(76) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(78) Exchange +(77) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(80) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(81) Exchange +(80) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(83) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(84) HashAggregate +(83) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(85) Exchange +(84) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) HashAggregate +(85) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(88) AdaptiveSparkPlan +(87) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt index c2b1c84e762d2..0dcdc424eb35e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt @@ -1,97 +1,101 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ ProjectExecTransformer (63) - +- ^ TopNTransformer (62) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) - :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) - :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (70) + +- ^ ProjectExecTransformer (68) + +- ^ TopNTransformer (67) + +- ^ InputIteratorTransformer (66) + +- ^ InputAdapter (65) + +- ^ ColumnarExchange (64) + +- ^ TopNTransformer (62) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner (58) + :- ^ InputIteratorTransformer (41) + : +- ^ InputAdapter (40) + : +- ^ ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ^ InputAdapter (33) + : +- ^ ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : :- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ^ InputAdapter (23) + : +- ^ ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + :- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (97) - +- HashAggregate (96) - +- HashAggregate (95) - +- Project (94) - +- ShuffledHashJoin Inner BuildRight (93) - :- Exchange (82) - : +- Project (81) - : +- ShuffledHashJoin Inner BuildLeft (80) - : :- Exchange (68) - : : +- Filter (67) - : : +- Scan parquet (66) - : +- Exchange (79) - : +- ShuffledHashJoin LeftSemi BuildRight (78) - : :- Exchange (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- Project (77) - : +- Filter (76) - : +- HashAggregate (75) - : +- Exchange (74) - : +- HashAggregate (73) - : +- Scan parquet (72) - +- ShuffledHashJoin LeftSemi BuildRight (92) - :- Exchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- Project (91) - +- Filter (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Scan parquet (86) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -357,180 +361,198 @@ Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], false -(63) ProjectExecTransformer +(63) WholeStageCodegenTransformer (X) +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: false + +(64) ColumnarExchange +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], [plan_id=X], [id=#X] + +(65) InputAdapter +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(66) InputIteratorTransformer +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(67) TopNTransformer +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true + +(68) ProjectExecTransformer Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(64) WholeStageCodegenTransformer (X) +(69) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(65) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(66) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(67) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(68) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(71) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(72) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(73) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(74) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(76) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(77) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(78) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(81) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(82) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(83) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(84) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(85) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(87) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(88) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(90) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(91) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(94) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(95) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(96) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(97) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(98) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt index 2e216f6c3e403..70d89d243c0a7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt @@ -1,61 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (60) +AdaptiveSparkPlan (65) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ ProjectExecTransformer (38) - +- ^ TopNTransformer (37) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (45) + +- ^ ProjectExecTransformer (43) + +- ^ TopNTransformer (42) + +- ^ InputIteratorTransformer (41) + +- ^ InputAdapter (40) + +- ^ ColumnarExchange (39) + +- ^ TopNTransformer (37) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (59) - +- HashAggregate (58) - +- HashAggregate (57) - +- Project (56) - +- ShuffledHashJoin Inner BuildRight (55) - :- Exchange (50) - : +- Project (49) - : +- ShuffledHashJoin Inner BuildLeft (48) - : :- Exchange (44) - : : +- Project (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Exchange (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + TakeOrderedAndProject (64) + +- HashAggregate (63) + +- HashAggregate (62) + +- Project (61) + +- ShuffledHashJoin Inner BuildRight (60) + :- Exchange (55) + : +- Project (54) + : +- ShuffledHashJoin Inner BuildLeft (53) + : :- Exchange (49) + : : +- Project (48) + : : +- Filter (47) + : : +- Scan parquet (46) + : +- Exchange (52) + : +- Filter (51) + : +- Scan parquet (50) + +- Exchange (59) + +- Project (58) + +- Filter (57) + +- Scan parquet (56) (1) Scan parquet @@ -215,110 +219,128 @@ Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((pr Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], false -(38) ProjectExecTransformer +(38) WholeStageCodegenTransformer (X) +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: false + +(39) ColumnarExchange +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] + +(40) InputAdapter +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(41) InputIteratorTransformer +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] + +(42) TopNTransformer +Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true + +(43) ProjectExecTransformer Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(39) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(40) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(41) Scan parquet +(46) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(42) Filter +(47) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(43) Project +(48) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(44) Exchange +(49) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Scan parquet +(50) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(46) Filter +(51) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(47) Exchange +(52) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) ShuffledHashJoin +(53) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(49) Project +(54) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(50) Exchange +(55) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(56) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(53) Project +(58) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(54) Exchange +(59) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) ShuffledHashJoin +(60) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(56) Project +(61) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(57) HashAggregate +(62) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(58) HashAggregate +(63) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(59) TakeOrderedAndProject +(64) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(60) AdaptiveSparkPlan +(65) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt index 4862e1cf53155..1f9d99a950b94 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt @@ -1,86 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (88) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (61) - +- ^ ProjectExecTransformer (59) - +- ^ TopNTransformer (58) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- ^ ProjectExecTransformer (64) + +- ^ TopNTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ^ InputAdapter (61) + +- ^ ColumnarExchange (60) + +- ^ TopNTransformer (58) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner (47) + :- ^ InputIteratorTransformer (38) + : +- ^ InputAdapter (37) + : +- ^ ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : :- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ NoopFilter (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (87) - +- HashAggregate (86) - +- Exchange (85) - +- HashAggregate (84) - +- Project (83) - +- ShuffledHashJoin Inner BuildRight (82) - :- Exchange (78) - : +- Project (77) - : +- ShuffledHashJoin Inner BuildRight (76) - : :- Exchange (71) - : : +- Project (70) - : : +- ShuffledHashJoin Inner BuildRight (69) - : : :- Exchange (64) - : : : +- Filter (63) - : : : +- Scan parquet (62) - : : +- Exchange (68) - : : +- Project (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (75) - : +- Project (74) - : +- Filter (73) - : +- Scan parquet (72) - +- Exchange (81) - +- Filter (80) - +- Scan parquet (79) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -325,145 +329,163 @@ Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], false -(59) ProjectExecTransformer +(59) WholeStageCodegenTransformer (X) +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: false + +(60) ColumnarExchange +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], [plan_id=X], [id=#X] + +(61) InputAdapter +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(62) InputIteratorTransformer +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] + +(63) TopNTransformer +Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] +Arguments: X, [revenue#X DESC NULLS LAST], true + +(64) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(60) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(61) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(62) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(63) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(64) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(67) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(68) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(70) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(71) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(72) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(73) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(74) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(75) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(76) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(77) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(78) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(80) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(81) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(83) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(84) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(85) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(88) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt index 69a74e597c448..677d0eec9c211 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt @@ -1,97 +1,101 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (103) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ ProjectExecTransformer (63) - +- ^ TopNTransformer (62) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) - :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) - :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (70) + +- ^ ProjectExecTransformer (68) + +- ^ TopNTransformer (67) + +- ^ InputIteratorTransformer (66) + +- ^ InputAdapter (65) + +- ^ ColumnarExchange (64) + +- ^ TopNTransformer (62) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner (58) + :- ^ InputIteratorTransformer (41) + : +- ^ InputAdapter (40) + : +- ^ ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ^ InputAdapter (33) + : +- ^ ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : :- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ NoopFilter (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ^ InputAdapter (23) + : +- ^ ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + :- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ NoopFilter (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (97) - +- HashAggregate (96) - +- HashAggregate (95) - +- Project (94) - +- ShuffledHashJoin Inner BuildRight (93) - :- Exchange (82) - : +- Project (81) - : +- ShuffledHashJoin Inner BuildLeft (80) - : :- Exchange (68) - : : +- Filter (67) - : : +- Scan parquet (66) - : +- Exchange (79) - : +- ShuffledHashJoin LeftSemi BuildRight (78) - : :- Exchange (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- Project (77) - : +- Filter (76) - : +- HashAggregate (75) - : +- Exchange (74) - : +- HashAggregate (73) - : +- Scan parquet (72) - +- ShuffledHashJoin LeftSemi BuildRight (92) - :- Exchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- Project (91) - +- Filter (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Scan parquet (86) + TakeOrderedAndProject (102) + +- HashAggregate (101) + +- HashAggregate (100) + +- Project (99) + +- ShuffledHashJoin Inner BuildRight (98) + :- Exchange (87) + : +- Project (86) + : +- ShuffledHashJoin Inner BuildLeft (85) + : :- Exchange (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Exchange (84) + : +- ShuffledHashJoin LeftSemi BuildRight (83) + : :- Exchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Project (82) + : +- Filter (81) + : +- HashAggregate (80) + : +- Exchange (79) + : +- HashAggregate (78) + : +- Scan parquet (77) + +- ShuffledHashJoin LeftSemi BuildRight (97) + :- Exchange (90) + : +- Filter (89) + : +- Scan parquet (88) + +- Project (96) + +- Filter (95) + +- HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Scan parquet (91) (1) Scan parquet @@ -361,184 +365,202 @@ Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], false -(63) ProjectExecTransformer +(63) WholeStageCodegenTransformer (X) +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: false + +(64) ColumnarExchange +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], [plan_id=X], [id=#X] + +(65) InputAdapter +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(66) InputIteratorTransformer +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] + +(67) TopNTransformer +Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] +Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true + +(68) ProjectExecTransformer Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(64) WholeStageCodegenTransformer (X) +(69) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(65) VeloxColumnarToRowExec +(70) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(66) Scan parquet +(71) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(67) Filter +(72) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(68) Exchange +(73) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(69) Scan parquet +(74) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(75) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(71) Exchange +(76) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(72) Scan parquet +(77) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(73) HashAggregate +(78) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(74) Exchange +(79) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) HashAggregate +(80) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(76) Filter +(81) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(77) Project +(82) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(78) ShuffledHashJoin +(83) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(79) Exchange +(84) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) ShuffledHashJoin +(85) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(81) Project +(86) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(82) Exchange +(87) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(83) Scan parquet +(88) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(84) Filter +(89) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(85) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(87) HashAggregate +(92) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(88) Exchange +(93) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(94) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(90) Filter +(95) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(91) Project +(96) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(93) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(94) Project +(99) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(95) HashAggregate +(100) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(96) HashAggregate +(101) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(97) TakeOrderedAndProject +(102) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(98) AdaptiveSparkPlan +(103) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt index b38bcc62c3d5f..5c9e3093b6f41 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt @@ -1,61 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (60) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (40) - +- ^ ProjectExecTransformer (38) - +- ^ TopNTransformer (37) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + TakeOrderedAndProject (39) + +- VeloxColumnarToRowExec (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ NoopFilter (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (59) - +- HashAggregate (58) - +- HashAggregate (57) - +- Project (56) - +- ShuffledHashJoin Inner BuildRight (55) - :- Exchange (50) - : +- Project (49) - : +- ShuffledHashJoin Inner BuildLeft (48) - : :- Exchange (44) - : : +- Project (43) - : : +- Filter (42) - : : +- Scan parquet (41) - : +- Exchange (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + TakeOrderedAndProject (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Project (55) + +- ShuffledHashJoin Inner BuildRight (54) + :- Exchange (49) + : +- Project (48) + : +- ShuffledHashJoin Inner BuildLeft (47) + : :- Exchange (43) + : : +- Project (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Filter (45) + : +- Scan parquet (44) + +- Exchange (53) + +- Project (52) + +- Filter (51) + +- Scan parquet (50) (1) Scan parquet @@ -213,116 +212,112 @@ Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedpric Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(37) TopNTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], false - -(38) ProjectExecTransformer -Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +(37) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: false -(39) WholeStageCodegenTransformer (X) +(38) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: false -(40) VeloxColumnarToRowExec +(39) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] +Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(41) Scan parquet +(40) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(42) Filter +(41) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(43) Project +(42) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(44) Exchange +(43) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Scan parquet +(44) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(46) Filter +(45) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(47) Exchange +(46) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(49) Project +(48) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(50) Exchange +(49) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(50) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(52) Filter +(51) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(53) Project +(52) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(54) Exchange +(53) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(56) Project +(55) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(57) HashAggregate +(56) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(58) HashAggregate +(57) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(59) TakeOrderedAndProject +(58) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(60) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt index 6993d4b584505..7b76ffb9abf57 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt @@ -1,90 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- ^ ProjectExecTransformer (64) - +- ^ TopNTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ^ InputAdapter (61) - +- ^ ColumnarExchange (60) - +- ^ TopNTransformer (58) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (61) + +- ^ ProjectExecTransformer (59) + +- ^ TopNTransformer (58) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner (47) + :- ^ InputIteratorTransformer (38) + : +- ^ InputAdapter (37) + : +- ^ ShuffleQueryStage (36) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : :- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ ShuffleQueryStage (21) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ FilterExecTransformer (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ ShuffleQueryStage (29) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ FilterExecTransformer (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- ShuffledHashJoin Inner BuildRight (87) - :- Exchange (83) - : +- Project (82) - : +- ShuffledHashJoin Inner BuildRight (81) - : :- Exchange (76) - : : +- Project (75) - : : +- ShuffledHashJoin Inner BuildRight (74) - : : :- Exchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- Exchange (73) - : : +- Project (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (80) - : +- Project (79) - : +- Filter (78) - : +- Scan parquet (77) - +- Exchange (86) - +- Filter (85) - +- Scan parquet (84) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- ShuffledHashJoin Inner BuildRight (82) + :- Exchange (78) + : +- Project (77) + : +- ShuffledHashJoin Inner BuildRight (76) + : :- Exchange (71) + : : +- Project (70) + : : +- ShuffledHashJoin Inner BuildRight (69) + : : :- Exchange (64) + : : : +- Filter (63) + : : : +- Scan parquet (62) + : : +- Exchange (68) + : : +- Project (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Exchange (75) + : +- Project (74) + : +- Filter (73) + : +- Scan parquet (72) + +- Exchange (81) + +- Filter (80) + +- Scan parquet (79) (1) Scan parquet @@ -326,160 +322,142 @@ Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], true -(59) WholeStageCodegenTransformer (X) -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: false - -(60) ColumnarExchange -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], [plan_id=X], [id=#X] - -(61) InputAdapter -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(62) InputIteratorTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(63) TopNTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: X, [revenue#X DESC NULLS LAST], true - -(64) ProjectExecTransformer +(59) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(65) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(66) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) Scan parquet +(62) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(63) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) Exchange +(64) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) Scan parquet +(65) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(66) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(67) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) Exchange +(68) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(75) Project +(70) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(76) Exchange +(71) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(72) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(78) Filter +(73) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(79) Project +(74) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(80) Exchange +(75) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(76) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(82) Project +(77) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(83) Exchange +(78) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(86) Exchange +(81) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(84) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(90) Exchange +(85) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(86) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(92) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt index d2e395032ac5d..4ac92af50a369 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt @@ -1,101 +1,97 @@ == Physical Plan == -AdaptiveSparkPlan (103) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (70) - +- ^ ProjectExecTransformer (68) - +- ^ TopNTransformer (67) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ ColumnarExchange (64) - +- ^ TopNTransformer (62) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) - :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) - :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51) - +- ReusedExchange (50) + VeloxColumnarToRowExec (65) + +- ^ ProjectExecTransformer (63) + +- ^ TopNTransformer (62) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner (58) + :- ^ InputIteratorTransformer (41) + : +- ^ InputAdapter (40) + : +- ^ ShuffleQueryStage (39) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ^ InputAdapter (33) + : +- ^ ShuffleQueryStage (32) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : :- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ FilterExecTransformer (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ^ InputAdapter (23) + : +- ^ ShuffleQueryStage (22) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + :- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ ShuffleQueryStage (47) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ FilterExecTransformer (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (102) - +- HashAggregate (101) - +- HashAggregate (100) - +- Project (99) - +- ShuffledHashJoin Inner BuildRight (98) - :- Exchange (87) - : +- Project (86) - : +- ShuffledHashJoin Inner BuildLeft (85) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (84) - : +- ShuffledHashJoin LeftSemi BuildRight (83) - : :- Exchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Project (82) - : +- Filter (81) - : +- HashAggregate (80) - : +- Exchange (79) - : +- HashAggregate (78) - : +- Scan parquet (77) - +- ShuffledHashJoin LeftSemi BuildRight (97) - :- Exchange (90) - : +- Filter (89) - : +- Scan parquet (88) - +- Project (96) - +- Filter (95) - +- HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Scan parquet (91) + TakeOrderedAndProject (97) + +- HashAggregate (96) + +- HashAggregate (95) + +- Project (94) + +- ShuffledHashJoin Inner BuildRight (93) + :- Exchange (82) + : +- Project (81) + : +- ShuffledHashJoin Inner BuildLeft (80) + : :- Exchange (68) + : : +- Filter (67) + : : +- Scan parquet (66) + : +- Exchange (79) + : +- ShuffledHashJoin LeftSemi BuildRight (78) + : :- Exchange (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- Project (77) + : +- Filter (76) + : +- HashAggregate (75) + : +- Exchange (74) + : +- HashAggregate (73) + : +- Scan parquet (72) + +- ShuffledHashJoin LeftSemi BuildRight (92) + :- Exchange (85) + : +- Filter (84) + : +- Scan parquet (83) + +- Project (91) + +- Filter (90) + +- HashAggregate (89) + +- Exchange (88) + +- HashAggregate (87) + +- Scan parquet (86) (1) Scan parquet @@ -361,198 +357,180 @@ Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true -(63) WholeStageCodegenTransformer (X) -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: false - -(64) ColumnarExchange -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], [plan_id=X], [id=#X] - -(65) InputAdapter -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(66) InputIteratorTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(67) TopNTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true - -(68) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(69) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(70) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(71) Scan parquet +(66) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(72) Filter +(67) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(73) Exchange +(68) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(69) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(75) Filter +(70) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(76) Exchange +(71) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(72) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(78) HashAggregate +(73) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(79) Exchange +(74) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) HashAggregate +(75) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(81) Filter +(76) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(82) Project +(77) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(83) ShuffledHashJoin +(78) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(84) Exchange +(79) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) ShuffledHashJoin +(80) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(86) Project +(81) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(87) Exchange +(82) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Scan parquet +(83) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(89) Filter +(84) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(90) Exchange +(85) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(86) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(92) HashAggregate +(87) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(93) Exchange +(88) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(89) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(95) Filter +(90) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(96) Project +(91) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(97) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(98) ShuffledHashJoin +(93) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(99) Project +(94) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(100) HashAggregate +(95) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(101) HashAggregate +(96) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(102) TakeOrderedAndProject +(97) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(103) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt index d25be3ae7b51b..417ef4d3471c4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt @@ -1,119 +1,115 @@ == Physical Plan == -AdaptiveSparkPlan (125) +AdaptiveSparkPlan (120) +- == Final Plan == - VeloxColumnarToRowExec (88) - +- ^ ProjectExecTransformer (86) - +- ^ TopNTransformer (85) - +- ^ InputIteratorTransformer (84) - +- ^ InputAdapter (83) - +- ^ ColumnarExchange (82) - +- ^ TopNTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner (70) - :- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) - : :- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ^ InputAdapter (22) - : : : +- ^ ShuffleQueryStage (21) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ^ InputAdapter (31) - : : +- ^ ShuffleQueryStage (30) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ^ InputAdapter (68) - +- ^ ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (83) + +- ^ ProjectExecTransformer (81) + +- ^ TopNTransformer (80) + +- ^ RegularHashAggregateExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ FlushableHashAggregateExecTransformer (72) + +- ^ ProjectExecTransformer (71) + +- ^ ShuffledHashJoinExecTransformer Inner (70) + :- ^ InputIteratorTransformer (61) + : +- ^ InputAdapter (60) + : +- ^ ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- ^ ProjectExecTransformer (56) + : +- ^ ShuffledHashJoinExecTransformer Inner (55) + : :- ^ InputIteratorTransformer (46) + : : +- ^ InputAdapter (45) + : : +- ^ ShuffleQueryStage (44) + : : +- ColumnarExchange (43) + : : +- ^ ProjectExecTransformer (41) + : : +- ^ ShuffledHashJoinExecTransformer Inner (40) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (39) + : : +- ^ InputAdapter (38) + : : +- ^ ShuffleQueryStage (37) + : : +- ColumnarExchange (36) + : : +- ^ ProjectExecTransformer (34) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) + : : : :- ^ InputIteratorTransformer (16) + : : : : +- ^ InputAdapter (15) + : : : : +- ^ ShuffleQueryStage (14) + : : : : +- ColumnarExchange (13) + : : : : +- ^ ProjectExecTransformer (11) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (23) + : : : +- ^ InputAdapter (22) + : : : +- ^ ShuffleQueryStage (21) + : : : +- ColumnarExchange (20) + : : : +- ^ ProjectExecTransformer (18) + : : : +- ^ Scan parquet (17) + : : +- ^ InputIteratorTransformer (32) + : : +- ^ InputAdapter (31) + : : +- ^ ShuffleQueryStage (30) + : : +- ColumnarExchange (29) + : : +- ^ ProjectExecTransformer (27) + : : +- ^ FilterExecTransformer (26) + : : +- ^ Scan parquet (25) + : +- ^ InputIteratorTransformer (54) + : +- ^ InputAdapter (53) + : +- ^ ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- ^ ProjectExecTransformer (49) + : +- ^ FilterExecTransformer (48) + : +- ^ Scan parquet (47) + +- ^ InputIteratorTransformer (69) + +- ^ InputAdapter (68) + +- ^ ShuffleQueryStage (67) + +- ColumnarExchange (66) + +- ^ ProjectExecTransformer (64) + +- ^ FilterExecTransformer (63) + +- ^ Scan parquet (62) +- == Initial Plan == - TakeOrderedAndProject (124) - +- HashAggregate (123) - +- Exchange (122) - +- HashAggregate (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin Inner BuildRight (112) - : :- Exchange (107) - : : +- Project (106) - : : +- ShuffledHashJoin Inner BuildLeft (105) - : : :- Exchange (91) - : : : +- Filter (90) - : : : +- Scan parquet (89) - : : +- Exchange (104) - : : +- ShuffledHashJoin LeftAnti BuildRight (103) - : : :- ShuffledHashJoin LeftSemi BuildRight (98) - : : : :- Exchange (95) - : : : : +- Project (94) - : : : : +- Filter (93) - : : : : +- Scan parquet (92) - : : : +- Exchange (97) - : : : +- Scan parquet (96) - : : +- Exchange (102) - : : +- Project (101) - : : +- Filter (100) - : : +- Scan parquet (99) - : +- Exchange (111) - : +- Project (110) - : +- Filter (109) - : +- Scan parquet (108) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + TakeOrderedAndProject (119) + +- HashAggregate (118) + +- Exchange (117) + +- HashAggregate (116) + +- Project (115) + +- ShuffledHashJoin Inner BuildRight (114) + :- Exchange (109) + : +- Project (108) + : +- ShuffledHashJoin Inner BuildRight (107) + : :- Exchange (102) + : : +- Project (101) + : : +- ShuffledHashJoin Inner BuildLeft (100) + : : :- Exchange (86) + : : : +- Filter (85) + : : : +- Scan parquet (84) + : : +- Exchange (99) + : : +- ShuffledHashJoin LeftAnti BuildRight (98) + : : :- ShuffledHashJoin LeftSemi BuildRight (93) + : : : :- Exchange (90) + : : : : +- Project (89) + : : : : +- Filter (88) + : : : : +- Scan parquet (87) + : : : +- Exchange (92) + : : : +- Scan parquet (91) + : : +- Exchange (97) + : : +- Project (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (106) + : +- Project (105) + : +- Filter (104) + : +- Scan parquet (103) + +- Exchange (113) + +- Project (112) + +- Filter (111) + +- Scan parquet (110) (1) Scan parquet @@ -444,207 +440,189 @@ Results [2]: [s_name#X, count(1)#X AS numwait#X] Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], true -(81) WholeStageCodegenTransformer (X) -Input [2]: [s_name#X, numwait#X] -Arguments: false - -(82) ColumnarExchange -Input [2]: [s_name#X, numwait#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [s_name#X, numwait#X], [plan_id=X], [id=#X] - -(83) InputAdapter -Input [2]: [s_name#X, numwait#X] - -(84) InputIteratorTransformer -Input [2]: [s_name#X, numwait#X] - -(85) TopNTransformer -Input [2]: [s_name#X, numwait#X] -Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], true - -(86) ProjectExecTransformer +(81) ProjectExecTransformer Output [2]: [s_name#X, numwait#X] Input [2]: [s_name#X, numwait#X] -(87) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(88) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(89) Scan parquet +(84) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(90) Filter +(85) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(91) Exchange +(86) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Scan parquet +(87) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(93) Filter +(88) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(94) Project +(89) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(91) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(97) Exchange +(92) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(93) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(99) Scan parquet +(94) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(100) Filter +(95) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(101) Project +(96) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(102) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(104) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(106) Project +(101) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(107) Exchange +(102) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(103) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(109) Filter +(104) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(110) Project +(105) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(111) Exchange +(106) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(113) Project +(108) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(114) Exchange +(109) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(111) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(117) Project +(112) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(113) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(120) Project +(115) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(121) HashAggregate +(116) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(122) Exchange +(117) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) HashAggregate +(118) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(124) TakeOrderedAndProject +(119) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(125) AdaptiveSparkPlan +(120) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt index f68bd6f226e03..0fd90816dbe13 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt @@ -1,65 +1,61 @@ == Physical Plan == -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (45) - +- ^ ProjectExecTransformer (43) - +- ^ TopNTransformer (42) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ ColumnarExchange (39) - +- ^ TopNTransformer (37) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ TopNTransformer (37) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ FilterExecTransformer (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (64) - +- HashAggregate (63) - +- HashAggregate (62) - +- Project (61) - +- ShuffledHashJoin Inner BuildRight (60) - :- Exchange (55) - : +- Project (54) - : +- ShuffledHashJoin Inner BuildLeft (53) - : :- Exchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- Exchange (52) - : +- Filter (51) - : +- Scan parquet (50) - +- Exchange (59) - +- Project (58) - +- Filter (57) - +- Scan parquet (56) + TakeOrderedAndProject (59) + +- HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Exchange (50) + : +- Project (49) + : +- ShuffledHashJoin Inner BuildLeft (48) + : :- Exchange (44) + : : +- Project (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Exchange (54) + +- Project (53) + +- Filter (52) + +- Scan parquet (51) (1) Scan parquet @@ -219,128 +215,110 @@ Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((pr Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true -(38) WholeStageCodegenTransformer (X) -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: false - -(39) ColumnarExchange -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] - -(40) InputAdapter -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(41) InputIteratorTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(42) TopNTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true - -(43) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(44) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(45) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(46) Scan parquet +(41) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(47) Filter +(42) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(48) Project +(43) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(49) Exchange +(44) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(45) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(51) Filter +(46) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(52) Exchange +(47) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(54) Project +(49) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(55) Exchange +(50) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Scan parquet +(51) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(57) Filter +(52) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(58) Project +(53) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(59) Exchange +(54) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(61) Project +(56) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(62) HashAggregate +(57) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(63) HashAggregate +(58) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(64) TakeOrderedAndProject +(59) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(65) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt index 94d539d1d4284..2343ce08aa39c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt @@ -1,90 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- ^ ProjectExecTransformer (64) - +- ^ TopNTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ^ InputAdapter (61) - +- ^ ColumnarExchange (60) - +- ^ TopNTransformer (58) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (61) + +- ^ ProjectExecTransformer (59) + +- ^ TopNTransformer (58) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner (47) + :- ^ InputIteratorTransformer (38) + : +- ^ InputAdapter (37) + : +- ^ ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : :- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ FilterExecTransformer (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ FilterExecTransformer (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- ShuffledHashJoin Inner BuildRight (87) - :- Exchange (83) - : +- Project (82) - : +- ShuffledHashJoin Inner BuildRight (81) - : :- Exchange (76) - : : +- Project (75) - : : +- ShuffledHashJoin Inner BuildRight (74) - : : :- Exchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- Exchange (73) - : : +- Project (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (80) - : +- Project (79) - : +- Filter (78) - : +- Scan parquet (77) - +- Exchange (86) - +- Filter (85) - +- Scan parquet (84) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- ShuffledHashJoin Inner BuildRight (82) + :- Exchange (78) + : +- Project (77) + : +- ShuffledHashJoin Inner BuildRight (76) + : :- Exchange (71) + : : +- Project (70) + : : +- ShuffledHashJoin Inner BuildRight (69) + : : :- Exchange (64) + : : : +- Filter (63) + : : : +- Scan parquet (62) + : : +- Exchange (68) + : : +- Project (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Exchange (75) + : +- Project (74) + : +- Filter (73) + : +- Scan parquet (72) + +- Exchange (81) + +- Filter (80) + +- Scan parquet (79) (1) Scan parquet @@ -326,160 +322,142 @@ Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], true -(59) WholeStageCodegenTransformer (X) -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: false - -(60) ColumnarExchange -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], [plan_id=X], [id=#X] - -(61) InputAdapter -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(62) InputIteratorTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(63) TopNTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: X, [revenue#X DESC NULLS LAST], true - -(64) ProjectExecTransformer +(59) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(65) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(66) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) Scan parquet +(62) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(63) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) Exchange +(64) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) Scan parquet +(65) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(66) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(67) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) Exchange +(68) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(75) Project +(70) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(76) Exchange +(71) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(72) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(78) Filter +(73) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(79) Project +(74) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(80) Exchange +(75) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(76) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(82) Project +(77) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(83) Exchange +(78) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(86) Exchange +(81) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(84) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(90) Exchange +(85) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(86) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(92) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt index d4cc8556872a2..f3cfa7a3b8c54 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt @@ -1,101 +1,97 @@ == Physical Plan == -AdaptiveSparkPlan (103) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (70) - +- ^ ProjectExecTransformer (68) - +- ^ TopNTransformer (67) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ ColumnarExchange (64) - +- ^ TopNTransformer (62) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) - :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) - :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (65) + +- ^ ProjectExecTransformer (63) + +- ^ TopNTransformer (62) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner (58) + :- ^ InputIteratorTransformer (41) + : +- ^ InputAdapter (40) + : +- ^ ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ^ InputAdapter (33) + : +- ^ ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : :- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ FilterExecTransformer (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ^ InputAdapter (23) + : +- ^ ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + :- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ FilterExecTransformer (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (102) - +- HashAggregate (101) - +- HashAggregate (100) - +- Project (99) - +- ShuffledHashJoin Inner BuildRight (98) - :- Exchange (87) - : +- Project (86) - : +- ShuffledHashJoin Inner BuildLeft (85) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (84) - : +- ShuffledHashJoin LeftSemi BuildRight (83) - : :- Exchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Project (82) - : +- Filter (81) - : +- HashAggregate (80) - : +- Exchange (79) - : +- HashAggregate (78) - : +- Scan parquet (77) - +- ShuffledHashJoin LeftSemi BuildRight (97) - :- Exchange (90) - : +- Filter (89) - : +- Scan parquet (88) - +- Project (96) - +- Filter (95) - +- HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Scan parquet (91) + TakeOrderedAndProject (97) + +- HashAggregate (96) + +- HashAggregate (95) + +- Project (94) + +- ShuffledHashJoin Inner BuildRight (93) + :- Exchange (82) + : +- Project (81) + : +- ShuffledHashJoin Inner BuildLeft (80) + : :- Exchange (68) + : : +- Filter (67) + : : +- Scan parquet (66) + : +- Exchange (79) + : +- ShuffledHashJoin LeftSemi BuildRight (78) + : :- Exchange (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- Project (77) + : +- Filter (76) + : +- HashAggregate (75) + : +- Exchange (74) + : +- HashAggregate (73) + : +- Scan parquet (72) + +- ShuffledHashJoin LeftSemi BuildRight (92) + :- Exchange (85) + : +- Filter (84) + : +- Scan parquet (83) + +- Project (91) + +- Filter (90) + +- HashAggregate (89) + +- Exchange (88) + +- HashAggregate (87) + +- Scan parquet (86) (1) Scan parquet @@ -361,198 +357,180 @@ Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true -(63) WholeStageCodegenTransformer (X) -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: false - -(64) ColumnarExchange -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], [plan_id=X], [id=#X] - -(65) InputAdapter -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(66) InputIteratorTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(67) TopNTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true - -(68) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(69) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(70) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(71) Scan parquet +(66) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(72) Filter +(67) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(73) Exchange +(68) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(69) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(75) Filter +(70) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(76) Exchange +(71) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(72) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(78) HashAggregate +(73) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(79) Exchange +(74) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) HashAggregate +(75) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(81) Filter +(76) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(82) Project +(77) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(83) ShuffledHashJoin +(78) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(84) Exchange +(79) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) ShuffledHashJoin +(80) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(86) Project +(81) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(87) Exchange +(82) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Scan parquet +(83) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(89) Filter +(84) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(90) Exchange +(85) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(86) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(92) HashAggregate +(87) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(93) Exchange +(88) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(89) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(95) Filter +(90) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(96) Project +(91) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(97) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(98) ShuffledHashJoin +(93) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(99) Project +(94) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(100) HashAggregate +(95) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(101) HashAggregate +(96) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(102) TakeOrderedAndProject +(97) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(103) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt index 3f578e901fdaf..8399bc8930204 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt @@ -1,65 +1,61 @@ == Physical Plan == -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (45) - +- ^ ProjectExecTransformer (43) - +- ^ TopNTransformer (42) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ ColumnarExchange (39) - +- ^ TopNTransformer (37) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ TopNTransformer (37) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ FilterExecTransformer (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (64) - +- HashAggregate (63) - +- HashAggregate (62) - +- Project (61) - +- ShuffledHashJoin Inner BuildRight (60) - :- Exchange (55) - : +- Project (54) - : +- ShuffledHashJoin Inner BuildLeft (53) - : :- Exchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- Exchange (52) - : +- Filter (51) - : +- Scan parquet (50) - +- Exchange (59) - +- Project (58) - +- Filter (57) - +- Scan parquet (56) + TakeOrderedAndProject (59) + +- HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Exchange (50) + : +- Project (49) + : +- ShuffledHashJoin Inner BuildLeft (48) + : :- Exchange (44) + : : +- Project (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Exchange (54) + +- Project (53) + +- Filter (52) + +- Scan parquet (51) (1) Scan parquet @@ -219,128 +215,110 @@ Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((pr Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true -(38) WholeStageCodegenTransformer (X) -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: false - -(39) ColumnarExchange -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] - -(40) InputAdapter -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(41) InputIteratorTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(42) TopNTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true - -(43) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(44) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(45) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(46) Scan parquet +(41) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(47) Filter +(42) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(48) Project +(43) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(49) Exchange +(44) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(45) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(51) Filter +(46) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(52) Exchange +(47) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(54) Project +(49) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(55) Exchange +(50) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Scan parquet +(51) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(57) Filter +(52) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(58) Project +(53) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(59) Exchange +(54) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(61) Project +(56) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(62) HashAggregate +(57) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(63) HashAggregate +(58) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(64) TakeOrderedAndProject +(59) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(65) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt index 6ee0ee0c0c070..70c516bbfa7a7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt @@ -1,90 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (66) - +- ^ ProjectExecTransformer (64) - +- ^ TopNTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ^ InputAdapter (61) - +- ^ ColumnarExchange (60) - +- ^ TopNTransformer (58) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (61) + +- ^ ProjectExecTransformer (59) + +- ^ TopNTransformer (58) + +- ^ ProjectExecTransformer (57) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ ShuffledHashJoinExecTransformer Inner (47) + :- ^ InputIteratorTransformer (38) + : +- ^ InputAdapter (37) + : +- ^ ShuffleQueryStage (36), Statistics(X) + : +- ColumnarExchange (35) + : +- ^ ProjectExecTransformer (33) + : +- ^ ShuffledHashJoinExecTransformer Inner (32) + : :- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ ShuffleQueryStage (21), Statistics(X) + : : +- ColumnarExchange (20) + : : +- ^ ProjectExecTransformer (18) + : : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : : :- ^ InputIteratorTransformer (8) + : : : +- ^ InputAdapter (7) + : : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : : +- ColumnarExchange (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ FilterExecTransformer (10) + : : +- ^ Scan parquet (9) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ ShuffleQueryStage (29), Statistics(X) + : +- ColumnarExchange (28) + : +- ^ ProjectExecTransformer (26) + : +- ^ FilterExecTransformer (25) + : +- ^ Scan parquet (24) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (92) - +- HashAggregate (91) - +- Exchange (90) - +- HashAggregate (89) - +- Project (88) - +- ShuffledHashJoin Inner BuildRight (87) - :- Exchange (83) - : +- Project (82) - : +- ShuffledHashJoin Inner BuildRight (81) - : :- Exchange (76) - : : +- Project (75) - : : +- ShuffledHashJoin Inner BuildRight (74) - : : :- Exchange (69) - : : : +- Filter (68) - : : : +- Scan parquet (67) - : : +- Exchange (73) - : : +- Project (72) - : : +- Filter (71) - : : +- Scan parquet (70) - : +- Exchange (80) - : +- Project (79) - : +- Filter (78) - : +- Scan parquet (77) - +- Exchange (86) - +- Filter (85) - +- Scan parquet (84) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- ShuffledHashJoin Inner BuildRight (82) + :- Exchange (78) + : +- Project (77) + : +- ShuffledHashJoin Inner BuildRight (76) + : :- Exchange (71) + : : +- Project (70) + : : +- ShuffledHashJoin Inner BuildRight (69) + : : :- Exchange (64) + : : : +- Filter (63) + : : : +- Scan parquet (62) + : : +- Exchange (68) + : : +- Project (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Exchange (75) + : +- Project (74) + : +- Filter (73) + : +- Scan parquet (72) + +- Exchange (81) + +- Filter (80) + +- Scan parquet (79) (1) Scan parquet @@ -329,163 +325,145 @@ Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], true -(59) WholeStageCodegenTransformer (X) -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: false - -(60) ColumnarExchange -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], [plan_id=X], [id=#X] - -(61) InputAdapter -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(62) InputIteratorTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] - -(63) TopNTransformer -Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -Arguments: X, [revenue#X DESC NULLS LAST], true - -(64) ProjectExecTransformer +(59) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(65) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(66) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) Scan parquet +(62) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(68) Filter +(63) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(69) Exchange +(64) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(70) Scan parquet +(65) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(71) Filter +(66) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(72) Project +(67) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(73) Exchange +(68) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(75) Project +(70) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(76) Exchange +(71) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(72) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(78) Filter +(73) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(79) Project +(74) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(80) Exchange +(75) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(76) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(82) Project +(77) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(83) Exchange +(78) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(86) Exchange +(81) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) Project +(83) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(89) HashAggregate +(84) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(90) Exchange +(85) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) HashAggregate +(86) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(92) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(93) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt index aac6f9f16d444..887c70c7bedc6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt @@ -1,101 +1,97 @@ == Physical Plan == -AdaptiveSparkPlan (103) +AdaptiveSparkPlan (98) +- == Final Plan == - VeloxColumnarToRowExec (70) - +- ^ ProjectExecTransformer (68) - +- ^ TopNTransformer (67) - +- ^ InputIteratorTransformer (66) - +- ^ InputAdapter (65) - +- ^ ColumnarExchange (64) - +- ^ TopNTransformer (62) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) - :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) - :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (65) + +- ^ ProjectExecTransformer (63) + +- ^ TopNTransformer (62) + +- ^ RegularHashAggregateExecTransformer (61) + +- ^ RegularHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ ShuffledHashJoinExecTransformer Inner (58) + :- ^ InputIteratorTransformer (41) + : +- ^ InputAdapter (40) + : +- ^ ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ ShuffledHashJoinExecTransformer Inner (35) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (34) + : +- ^ InputAdapter (33) + : +- ^ ShuffleQueryStage (32), Statistics(X) + : +- ColumnarExchange (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) + : :- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ ShuffleQueryStage (14), Statistics(X) + : : +- ColumnarExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ FilterExecTransformer (10) + : : +- ^ Scan parquet (9) + : +- ^ ProjectExecTransformer (27) + : +- ^ FilterExecTransformer (26) + : +- ^ RegularHashAggregateExecTransformer (25) + : +- ^ InputIteratorTransformer (24) + : +- ^ InputAdapter (23) + : +- ^ ShuffleQueryStage (22), Statistics(X) + : +- ColumnarExchange (21) + : +- ^ ProjectExecTransformer (19) + : +- ^ FlushableHashAggregateExecTransformer (18) + : +- ^ Scan parquet (17) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) + :- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ ShuffleQueryStage (47), Statistics(X) + : +- ColumnarExchange (46) + : +- ^ ProjectExecTransformer (44) + : +- ^ FilterExecTransformer (43) + : +- ^ Scan parquet (42) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51), Statistics(X) + +- ReusedExchange (50) +- == Initial Plan == - TakeOrderedAndProject (102) - +- HashAggregate (101) - +- HashAggregate (100) - +- Project (99) - +- ShuffledHashJoin Inner BuildRight (98) - :- Exchange (87) - : +- Project (86) - : +- ShuffledHashJoin Inner BuildLeft (85) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (84) - : +- ShuffledHashJoin LeftSemi BuildRight (83) - : :- Exchange (76) - : : +- Filter (75) - : : +- Scan parquet (74) - : +- Project (82) - : +- Filter (81) - : +- HashAggregate (80) - : +- Exchange (79) - : +- HashAggregate (78) - : +- Scan parquet (77) - +- ShuffledHashJoin LeftSemi BuildRight (97) - :- Exchange (90) - : +- Filter (89) - : +- Scan parquet (88) - +- Project (96) - +- Filter (95) - +- HashAggregate (94) - +- Exchange (93) - +- HashAggregate (92) - +- Scan parquet (91) + TakeOrderedAndProject (97) + +- HashAggregate (96) + +- HashAggregate (95) + +- Project (94) + +- ShuffledHashJoin Inner BuildRight (93) + :- Exchange (82) + : +- Project (81) + : +- ShuffledHashJoin Inner BuildLeft (80) + : :- Exchange (68) + : : +- Filter (67) + : : +- Scan parquet (66) + : +- Exchange (79) + : +- ShuffledHashJoin LeftSemi BuildRight (78) + : :- Exchange (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- Project (77) + : +- Filter (76) + : +- HashAggregate (75) + : +- Exchange (74) + : +- HashAggregate (73) + : +- Scan parquet (72) + +- ShuffledHashJoin LeftSemi BuildRight (92) + :- Exchange (85) + : +- Filter (84) + : +- Scan parquet (83) + +- Project (91) + +- Filter (90) + +- HashAggregate (89) + +- Exchange (88) + +- HashAggregate (87) + +- Scan parquet (86) (1) Scan parquet @@ -365,202 +361,184 @@ Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true -(63) WholeStageCodegenTransformer (X) -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: false - -(64) ColumnarExchange -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], [plan_id=X], [id=#X] - -(65) InputAdapter -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(66) InputIteratorTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] - -(67) TopNTransformer -Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true - -(68) ProjectExecTransformer +(63) ProjectExecTransformer Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(69) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(70) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(71) Scan parquet +(66) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(72) Filter +(67) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(73) Exchange +(68) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(69) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(75) Filter +(70) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(76) Exchange +(71) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(77) Scan parquet +(72) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(78) HashAggregate +(73) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(79) Exchange +(74) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(80) HashAggregate +(75) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(81) Filter +(76) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(82) Project +(77) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(83) ShuffledHashJoin +(78) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(84) Exchange +(79) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) ShuffledHashJoin +(80) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(86) Project +(81) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(87) Exchange +(82) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) Scan parquet +(83) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(89) Filter +(84) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(90) Exchange +(85) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) Scan parquet +(86) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(92) HashAggregate +(87) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(93) Exchange +(88) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) HashAggregate +(89) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(95) Filter +(90) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(96) Project +(91) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(97) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(98) ShuffledHashJoin +(93) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(99) Project +(94) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(100) HashAggregate +(95) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(101) HashAggregate +(96) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(102) TakeOrderedAndProject +(97) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(103) AdaptiveSparkPlan +(98) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt index a036140600558..0d204454a99d9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt @@ -1,65 +1,61 @@ == Physical Plan == -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (60) +- == Final Plan == - VeloxColumnarToRowExec (45) - +- ^ ProjectExecTransformer (43) - +- ^ TopNTransformer (42) - +- ^ InputIteratorTransformer (41) - +- ^ InputAdapter (40) - +- ^ ColumnarExchange (39) - +- ^ TopNTransformer (37) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (40) + +- ^ ProjectExecTransformer (38) + +- ^ TopNTransformer (37) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ RegularHashAggregateExecTransformer (34) + +- ^ ProjectExecTransformer (33) + +- ^ ShuffledHashJoinExecTransformer Inner (32) + :- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ ShuffleQueryStage (21), Statistics(X) + : +- ColumnarExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ ShuffledHashJoinExecTransformer Inner (17) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ ShuffleQueryStage (6), Statistics(X) + : : +- ColumnarExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (16) + : +- ^ InputAdapter (15) + : +- ^ ShuffleQueryStage (14), Statistics(X) + : +- ColumnarExchange (13) + : +- ^ ProjectExecTransformer (11) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ FilterExecTransformer (25) + +- ^ Scan parquet (24) +- == Initial Plan == - TakeOrderedAndProject (64) - +- HashAggregate (63) - +- HashAggregate (62) - +- Project (61) - +- ShuffledHashJoin Inner BuildRight (60) - :- Exchange (55) - : +- Project (54) - : +- ShuffledHashJoin Inner BuildLeft (53) - : :- Exchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- Exchange (52) - : +- Filter (51) - : +- Scan parquet (50) - +- Exchange (59) - +- Project (58) - +- Filter (57) - +- Scan parquet (56) + TakeOrderedAndProject (59) + +- HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Exchange (50) + : +- Project (49) + : +- ShuffledHashJoin Inner BuildLeft (48) + : :- Exchange (44) + : : +- Project (43) + : : +- Filter (42) + : : +- Scan parquet (41) + : +- Exchange (47) + : +- Filter (46) + : +- Scan parquet (45) + +- Exchange (54) + +- Project (53) + +- Filter (52) + +- Scan parquet (51) (1) Scan parquet @@ -221,130 +217,112 @@ Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice# Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true -(38) WholeStageCodegenTransformer (X) -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: false - -(39) ColumnarExchange -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: SinglePartition, ENSURE_REQUIREMENTS, [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] - -(40) InputAdapter -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(41) InputIteratorTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] - -(42) TopNTransformer -Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], true - -(43) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(44) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(45) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(46) Scan parquet +(41) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(47) Filter +(42) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(48) Project +(43) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(49) Exchange +(44) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(45) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(51) Filter +(46) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(52) Exchange +(47) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) ShuffledHashJoin +(48) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(54) Project +(49) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(55) Exchange +(50) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(56) Scan parquet +(51) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(57) Filter +(52) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(58) Project +(53) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(59) Exchange +(54) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(61) Project +(56) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(62) HashAggregate +(57) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(63) HashAggregate +(58) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(64) TakeOrderedAndProject +(59) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(65) AdaptiveSparkPlan +(60) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file