diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000000..ab71b593784a --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +backends-velox/src/test/resources/tpch-approved-plan/** linguist-generated diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/FilterExecTransformer.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/FilterExecTransformer.scala index 7d399308ec02..daa08498bee3 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/FilterExecTransformer.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/FilterExecTransformer.scala @@ -21,6 +21,8 @@ import org.apache.spark.sql.execution.SparkPlan case class FilterExecTransformer(condition: Expression, child: SparkPlan) extends FilterExecTransformerBase(condition, child) { + // FIXME: Should use field "condition" to store the actual executed filter expressions. + // To make optimization easier (like to remove filter when it actually does nothing) override protected def getRemainingCondition: Expression = { val scanFilters = child match { // Get the filters including the manually pushed down ones. diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt index 7a80ed95f3e8..699c17ea4562 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt @@ -1,31 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (27) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (18) + +- ^ SortExecTransformer (16) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7) + +- ColumnarExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (3) + +- ^ ProjectExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (26) + +- Exchange (25) + +- HashAggregate (24) + +- Exchange (23) + +- HashAggregate (22) + +- Project (21) + +- Filter (20) + +- Scan parquet (19) (1) Scan parquet @@ -35,120 +34,116 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) FilterExecTransformer -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Arguments: (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(3) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) ProjectExecTransformer +(4) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(6) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(6) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(7) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(8) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(9) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(12) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(13) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(14) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(15) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(16) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(18) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(19) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(20) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(21) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(22) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(23) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(24) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(25) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(26) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(27) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt index 935650e26059..5b7e141a4beb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt @@ -1,68 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (39) + +- TakeOrderedAndProjectExecTransformer (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- ^ ProjectExecTransformer (29) + +- ^ FlushableHashAggregateExecTransformer (28) + +- ^ ProjectExecTransformer (27) + +- ^ BroadcastHashJoinExecTransformer Inner (26) + :- ^ ProjectExecTransformer (19) + : +- ^ BroadcastHashJoinExecTransformer Inner (18) + : :- ^ ProjectExecTransformer (10) + : : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : : :- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ Scan parquet (2) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ BroadcastQueryStage (15) + : +- ColumnarBroadcastExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ Scan parquet (11) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ BroadcastQueryStage (23) + +- ColumnarBroadcastExchange (22) + +- ^ Scan parquet (20) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- BroadcastHashJoin Inner BuildRight (57) + :- Project (53) + : +- BroadcastHashJoin Inner BuildRight (52) + : :- Project (47) + : : +- BroadcastHashJoin Inner BuildRight (46) + : : :- Filter (41) + : : : +- Scan parquet (40) + : : +- BroadcastExchange (45) + : : +- Project (44) + : : +- Filter (43) + : : +- Scan parquet (42) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- BroadcastExchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,296 +68,280 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) Scan parquet +(2) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(4) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(5) ProjectExecTransformer +(3) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(6) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(7) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(8) BroadcastQueryStage +(6) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(9) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(10) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(13) Scan parquet +(11) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) - -(15) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(16) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(17) ColumnarBroadcastExchange +(14) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(18) BroadcastQueryStage +(15) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(19) InputAdapter +(16) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(21) BroadcastHashJoinExecTransformer +(18) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(22) ProjectExecTransformer +(19) ProjectExecTransformer Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) Scan parquet +(20) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(24) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(25) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(26) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(27) BroadcastQueryStage +(23) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(28) InputAdapter +(24) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(29) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(30) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(31) ProjectExecTransformer +(27) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(28) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(33) ProjectExecTransformer +(29) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(34) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(31) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(32) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(33) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(34) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(40) ProjectExecTransformer +(36) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(41) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(38) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(40) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(41) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(45) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(47) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(48) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(49) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(50) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(51) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(53) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(55) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(56) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(59) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(60) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(61) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt index 1cc9e9ac278e..32cb5ccf87b3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt @@ -1,59 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ^ InputAdapter (34) - +- ^ ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- ^ InputAdapter (8) - : +- ^ BroadcastQueryStage (7) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ BroadcastQueryStage (17) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30) + +- ColumnarExchange (29) + +- ^ FilterExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ BroadcastHashJoinExecTransformer Inner (17) + :- ^ ProjectExecTransformer (9) + : +- ^ BroadcastHashJoinExecTransformer Inner (8) + : :- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ BroadcastQueryStage (14) + +- ColumnarBroadcastExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (54) + +- Exchange (53) + +- Filter (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- Project (42) + : +- BroadcastHashJoin Inner BuildRight (41) + : :- Filter (37) + : : +- Scan parquet (36) + : +- BroadcastExchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- BroadcastExchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -63,252 +60,240 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(3) Scan parquet +(2) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(4) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(5) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(7) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(8) InputAdapter +(6) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(9) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(10) BroadcastHashJoinExecTransformer +(8) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(12) Scan parquet +(10) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(13) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) - -(14) ProjectExecTransformer +(11) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [1]: [n_nationkey#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [n_nationkey#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(21) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(22) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(23) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(24) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(22) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(24) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(30) FilterExecTransformer +(27) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(29) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(31) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(33) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(36) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(37) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(38) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(39) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(40) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(42) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(43) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(44) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(45) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(46) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(48) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(49) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(50) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(51) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(55) Filter +(52) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(53) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(54) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt index 1d477cbd774e..a9cea181da0f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt @@ -1,44 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (26) + +- ^ SortExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer Inner (9) + :- ^ InputIteratorTransformer (6) + : +- ^ InputAdapter (5) + : +- ^ BroadcastQueryStage (4) + : +- ColumnarBroadcastExchange (3) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (8) + +- ^ Scan parquet (7) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin Inner BuildLeft (33) + :- BroadcastExchange (29) + : +- Filter (28) + : +- Scan parquet (27) + +- Project (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -48,182 +46,174 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: isnotnull(o_orderkey#X) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(7) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) - -(10) ProjectExecTransformer +(8) ProjectExecTransformer Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(13) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(14) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(15) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(16) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(24) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(28) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(29) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(30) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(31) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(32) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(34) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(35) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(36) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(37) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(38) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(39) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt index 492cfafb1ec1..1963624ed312 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt @@ -1,53 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (51) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftOuter (9) :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ ProjectExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (50) + +- Exchange (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- Exchange (44) + +- HashAggregate (43) + +- Project (42) + +- BroadcastHashJoin LeftOuter BuildRight (41) + :- Scan parquet (36) + +- BroadcastExchange (40) + +- Project (39) + +- Filter (38) + +- Scan parquet (37) (1) Scan parquet @@ -63,224 +62,220 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(3) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) - -(4) ProjectExecTransformer +(3) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(5) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(7) BroadcastQueryStage +(6) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(8) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(9) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(10) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(11) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(12) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(13) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, count#X] Input [2]: [c_custkey#X, count#X] -(14) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(14) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(16) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(19) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(21) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(25) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(29) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(31) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(33) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(36) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(37) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(38) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(39) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(40) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(42) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(43) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(44) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(46) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(47) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(48) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(49) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(50) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(51) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt index b564240ff552..1d64bb80886d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt @@ -1,37 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (32) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (20) + +- ^ ProjectExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer Inner (9) + :- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (31) + +- Exchange (30) + +- HashAggregate (29) + +- Project (28) + +- BroadcastHashJoin Inner BuildRight (27) + :- Project (23) + : +- Filter (22) + : +- Scan parquet (21) + +- BroadcastExchange (26) + +- Filter (25) + +- Scan parquet (24) (1) Scan parquet @@ -41,152 +39,144 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) Scan parquet +(3) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(5) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: isnotnull(p_partkey#X) - -(6) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [p_partkey#X, p_type#X] Arguments: false -(7) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(8) BroadcastQueryStage +(6) BroadcastQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(9) InputAdapter +(7) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(10) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(13) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(15) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(21) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(22) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(23) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(24) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(25) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(26) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(27) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(28) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(29) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(30) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(31) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(32) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt index ff632e930f72..d8563db5b941 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt @@ -1,46 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ BroadcastHashJoinExecTransformer Inner (18) + :- ^ InputIteratorTransformer (6) + : +- ^ InputAdapter (5) + : +- ^ BroadcastQueryStage (4) + : +- ColumnarBroadcastExchange (3) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (17) + +- ^ RegularHashAggregateExecTransformer (16) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ FlushableHashAggregateExecTransformer (9) + +- ^ ProjectExecTransformer (8) + +- ^ Scan parquet (7) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- Project (41) - +- BroadcastHashJoin Inner BuildLeft (40) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Filter (39) - +- HashAggregate (38) - +- Exchange (37) - +- HashAggregate (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (41) + +- Exchange (40) + +- Project (39) + +- BroadcastHashJoin Inner BuildLeft (38) + :- BroadcastExchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -50,190 +48,182 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: isnotnull(s_suppkey#X) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) Scan parquet +(7) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) - -(10) ProjectExecTransformer +(8) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) FlushableHashAggregateExecTransformer +(9) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(12) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(13) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(14) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(15) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(16) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(19) FilterExecTransformer +(17) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(18) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) SortExecTransformer +(25) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(29) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) BroadcastExchange +(30) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(34) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(35) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(36) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(37) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(39) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(40) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(41) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(42) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt index 06fcb53dfebe..d8664006f507 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt @@ -1,57 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (62) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (41) + +- ^ SortExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36) + +- ColumnarExchange (35) + +- ^ RegularHashAggregateExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30) + +- ColumnarExchange (29) + +- ^ ProjectExecTransformer (27) + +- ^ FlushableHashAggregateExecTransformer (26) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ BroadcastHashJoinExecTransformer Inner (16) + :- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + : :- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ BroadcastQueryStage (6) + : +- ColumnarBroadcastExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ BroadcastQueryStage (13) + +- ColumnarBroadcastExchange (12) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (61) + +- Exchange (60) + +- HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- BroadcastHashJoin LeftAnti BuildRight (48) + : :- Filter (43) + : : +- Scan parquet (42) + : +- BroadcastExchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- BroadcastExchange (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -61,252 +66,278 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: isnotnull(ps_partkey#X) +(2) Scan parquet +Output [2]: [s_suppkey#X, s_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_comment)] +ReadSchema: struct + +(3) ProjectExecTransformer +Output [1]: [s_suppkey#X] +Input [2]: [s_suppkey#X, s_comment#X] + +(4) WholeStageCodegenTransformer (X) +Input [1]: [s_suppkey#X] +Arguments: false + +(5) ColumnarBroadcastExchange +Input [1]: [s_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] + +(6) BroadcastQueryStage +Output [1]: [s_suppkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [s_suppkey#X] -(3) Scan parquet +(8) InputIteratorTransformer +Input [1]: [s_suppkey#X] + +(9) BroadcastHashJoinExecTransformer +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join condition: None + +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(4) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) - -(5) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(6) ColumnarBroadcastExchange +(12) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(7) BroadcastQueryStage +(13) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(8) InputAdapter +(14) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(9) InputIteratorTransformer +(15) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(10) BroadcastHashJoinExecTransformer +(16) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(11) ProjectExecTransformer +(17) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(13) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(14) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(21) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(22) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(23) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(24) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(26) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(27) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(29) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(30) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(31) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(32) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(33) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(35) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(37) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(39) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(41) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(42) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(43) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(45) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(46) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(47) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(49) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(50) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(51) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(53) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(54) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(57) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(58) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(59) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(61) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(62) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt index 32359df26265..468f95bff9fd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt @@ -1,86 +1,83 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (83) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32) - +- ReusedExchange (31) + VeloxColumnarToRowExec (50) + +- TakeOrderedAndProjectExecTransformer (49) + +- ^ RegularHashAggregateExecTransformer (47) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FlushableHashAggregateExecTransformer (40) + +- ^ ProjectExecTransformer (39) + +- ^ BroadcastHashJoinExecTransformer Inner (38) + :- ^ ProjectExecTransformer (26) + : +- ^ BroadcastHashJoinExecTransformer Inner (25) + : :- ^ InputIteratorTransformer (6) + : : +- ^ InputAdapter (5) + : : +- ^ BroadcastQueryStage (4) + : : +- ColumnarBroadcastExchange (3) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (24) + : :- ^ Scan parquet (7) + : +- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ BroadcastQueryStage (21) + : +- ColumnarBroadcastExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ FilterExecTransformer (17) + : +- ^ RegularHashAggregateExecTransformer (16) + : +- ^ InputIteratorTransformer (15) + : +- ^ InputAdapter (14) + : +- ^ ShuffleQueryStage (13) + : +- ColumnarExchange (12) + : +- ^ ProjectExecTransformer (10) + : +- ^ FlushableHashAggregateExecTransformer (9) + : +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (37) + +- ^ InputAdapter (36) + +- ^ BroadcastQueryStage (35) + +- ColumnarBroadcastExchange (34) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (32) + :- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ BroadcastQueryStage (29) + +- ReusedExchange (28) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (82) + +- HashAggregate (81) + +- Exchange (80) + +- HashAggregate (79) + +- Project (78) + +- BroadcastHashJoin Inner BuildRight (77) + :- Project (65) + : +- BroadcastHashJoin Inner BuildLeft (64) + : :- BroadcastExchange (53) + : : +- Filter (52) + : : +- Scan parquet (51) + : +- BroadcastHashJoin LeftSemi BuildRight (63) + : :- Filter (55) + : : +- Scan parquet (54) + : +- BroadcastExchange (62) + : +- Project (61) + : +- Filter (60) + : +- HashAggregate (59) + : +- Exchange (58) + : +- HashAggregate (57) + : +- Scan parquet (56) + +- BroadcastExchange (76) + +- BroadcastHashJoin LeftSemi BuildRight (75) + :- Filter (67) + : +- Scan parquet (66) + +- BroadcastExchange (74) + +- Project (73) + +- Filter (72) + +- HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Scan parquet (68) (1) Scan parquet @@ -90,387 +87,375 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_name#X] -Arguments: isnotnull(c_custkey#X) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_name#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(8) Scan parquet +(7) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) - -(10) Scan parquet +(8) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(9) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(13) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(12) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(13) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(14) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(15) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(16) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(17) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(20) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(21) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(24) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(25) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(26) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(27) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) FilterExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: isnotnull(l_orderkey#X) - -(31) ReusedExchange [Reuses operator id: 22] +(28) ReusedExchange [Reuses operator id: 20] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(29) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(32) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(36) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(40) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(43) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(44) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(45) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(46) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(47) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(49) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(51) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(52) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(53) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(54) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(55) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(56) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(57) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(58) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(59) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(60) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(61) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(62) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(63) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(64) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(65) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(67) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(68) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(69) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(70) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(71) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(72) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(73) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(74) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(76) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(78) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(79) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(80) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(81) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(82) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(83) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt index 232c27fb8a9e..1be7fcaa15c3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt @@ -1,36 +1,34 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (31) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (19) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer Inner (9) + :- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (30) + +- Exchange (29) + +- HashAggregate (28) + +- Project (27) + +- BroadcastHashJoin Inner BuildRight (26) + :- Project (22) + : +- Filter (21) + : +- Scan parquet (20) + +- BroadcastExchange (25) + +- Filter (24) + +- Scan parquet (23) (1) Scan parquet @@ -40,148 +38,140 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) FilterExecTransformer -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Arguments: ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(4) Scan parquet +(3) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(5) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) - -(6) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(7) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(8) BroadcastQueryStage +(6) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(9) InputAdapter +(7) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(10) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(13) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(15) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(20) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(21) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(22) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(23) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(24) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(25) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(26) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(27) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(28) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(29) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(30) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(33) AdaptiveSparkPlan +(31) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt index fe2ae1df7cc7..d19495f88198 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt @@ -1,105 +1,103 @@ == Physical Plan == -AdaptiveSparkPlan (107) +AdaptiveSparkPlan (105) +- == Final Plan == - VeloxColumnarToRowExec (70) - +- ^ SortExecTransformer (68) - +- ^ InputIteratorTransformer (67) - +- ^ InputAdapter (66) - +- ^ ShuffleQueryStage (65) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (51) - : :- ^ InputIteratorTransformer (9) - : : +- ^ InputAdapter (8) - : : +- ^ AQEShuffleRead (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ BroadcastQueryStage (48) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner (44) - : :- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (20) - : : :- ^ FilterExecTransformer (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- ^ InputAdapter (18) - : : +- ^ BroadcastQueryStage (17) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ FilterExecTransformer (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ^ InputAdapter (39) - : +- ^ ShuffleQueryStage (38) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ FilterExecTransformer (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- ^ InputAdapter (59) - +- ^ BroadcastQueryStage (58) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ FilterExecTransformer (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (68) + +- ^ SortExecTransformer (66) + +- ^ InputIteratorTransformer (65) + +- ^ InputAdapter (64) + +- ^ ShuffleQueryStage (63) + +- ColumnarExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ BroadcastHashJoinExecTransformer Inner (59) + :- ^ ProjectExecTransformer (51) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (50) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ AQEShuffleRead (6) + : : +- ^ ShuffleQueryStage (5) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ BroadcastQueryStage (47) + : +- ColumnarBroadcastExchange (46) + : +- AQEShuffleRead (45) + : +- ShuffleQueryStage (44) + : +- ColumnarExchange (43) + : +- ^ ProjectExecTransformer (41) + : +- ^ BroadcastHashJoinExecTransformer Inner (40) + : :- ^ InputIteratorTransformer (22) + : : +- ^ InputAdapter (21) + : : +- ^ BroadcastQueryStage (20) + : : +- ColumnarBroadcastExchange (19) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : :- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ BroadcastQueryStage (14) + : : +- ColumnarBroadcastExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ FilterExecTransformer (39) + : +- ^ ProjectExecTransformer (38) + : +- ^ RegularHashAggregateExecTransformer (37) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ ShuffleQueryStage (34) + : +- ColumnarExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FlushableHashAggregateExecTransformer (30) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (29) + : :- ^ ProjectExecTransformer (24) + : : +- ^ Scan parquet (23) + : +- ^ InputIteratorTransformer (28) + : +- ^ InputAdapter (27) + : +- ^ BroadcastQueryStage (26) + : +- ReusedExchange (25) + +- ^ InputIteratorTransformer (58) + +- ^ InputAdapter (57) + +- ^ BroadcastQueryStage (56) + +- ColumnarBroadcastExchange (55) + +- ^ ProjectExecTransformer (53) + +- ^ Scan parquet (52) +- == Initial Plan == - Sort (106) - +- Exchange (105) - +- Project (104) - +- BroadcastHashJoin Inner BuildRight (103) - :- Project (98) - : +- ShuffledHashJoin LeftSemi BuildRight (97) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (96) - : +- Project (95) - : +- BroadcastHashJoin Inner BuildLeft (94) - : :- BroadcastExchange (81) - : : +- BroadcastHashJoin LeftSemi BuildRight (80) - : : :- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (79) - : : +- Project (78) - : : +- Filter (77) - : : +- Scan parquet (76) - : +- Filter (93) - : +- HashAggregate (92) - : +- Exchange (91) - : +- HashAggregate (90) - : +- BroadcastHashJoin LeftSemi BuildRight (89) - : :- Project (84) - : : +- Filter (83) - : : +- Scan parquet (82) - : +- BroadcastExchange (88) - : +- Project (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (102) - +- Project (101) - +- Filter (100) - +- Scan parquet (99) + Sort (104) + +- Exchange (103) + +- Project (102) + +- BroadcastHashJoin Inner BuildRight (101) + :- Project (96) + : +- ShuffledHashJoin LeftSemi BuildRight (95) + : :- Exchange (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- Exchange (94) + : +- Project (93) + : +- BroadcastHashJoin Inner BuildLeft (92) + : :- BroadcastExchange (79) + : : +- BroadcastHashJoin LeftSemi BuildRight (78) + : : :- Filter (73) + : : : +- Scan parquet (72) + : : +- BroadcastExchange (77) + : : +- Project (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- Filter (91) + : +- HashAggregate (90) + : +- Exchange (89) + : +- HashAggregate (88) + : +- BroadcastHashJoin LeftSemi BuildRight (87) + : :- Project (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (86) + : +- Project (85) + : +- Filter (84) + : +- Scan parquet (83) + +- BroadcastExchange (100) + +- Project (99) + +- Filter (98) + +- Scan parquet (97) (1) Scan parquet @@ -109,460 +107,452 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: isnotnull(s_nationkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(6) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(9) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) - -(12) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) - -(14) ProjectExecTransformer +(11) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(19) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(20) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(21) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(22) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(23) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) FilterExecTransformer -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) - -(28) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(25) ReusedExchange [Reuses operator id: 13] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(26) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(27) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(28) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(33) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(34) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(35) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(36) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(39) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(40) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer -Output [1]: [ps_suppkey#X] +(41) ProjectExecTransformer +Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) -Input [1]: [ps_suppkey#X] +(42) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(43) ColumnarExchange +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] + +(44) ShuffleQueryStage +Output [1]: [ps_suppkey#X] +Arguments: X + +(45) AQEShuffleRead +Input [1]: [ps_suppkey#X] +Arguments: local + +(46) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(47) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(48) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(49) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(50) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(51) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(52) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) - -(55) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(55) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(56) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(57) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(58) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(59) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(60) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(62) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(63) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) InputAdapter +(64) InputAdapter Input [2]: [s_name#X, s_address#X] -(67) InputIteratorTransformer +(65) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(68) SortExecTransformer +(66) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(69) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(70) VeloxColumnarToRowExec +(68) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(71) Scan parquet +(69) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(72) Filter +(70) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(73) Exchange +(71) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(72) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(75) Filter +(73) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(76) Scan parquet +(74) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(77) Filter +(75) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(78) Project +(76) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(79) BroadcastExchange +(77) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(81) BroadcastExchange +(79) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) Scan parquet +(80) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(83) Filter +(81) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(84) Project +(82) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(85) Scan parquet +(83) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(84) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(85) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) BroadcastExchange +(86) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(90) HashAggregate +(88) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(91) Exchange +(89) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(90) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Filter +(91) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(94) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(95) Project +(93) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(96) Exchange +(94) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(95) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(98) Project +(96) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(99) Scan parquet +(97) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(100) Filter +(98) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(101) Project +(99) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(102) BroadcastExchange +(100) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(103) BroadcastHashJoin +(101) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(102) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(105) Exchange +(103) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Sort +(104) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(107) AdaptiveSparkPlan +(105) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt index c3bd62e04735..faf7f5799f34 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt @@ -1,91 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (87) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- TakeOrderedAndProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ BroadcastQueryStage (5) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (54) + +- TakeOrderedAndProjectExecTransformer (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ^ InputAdapter (49) + +- ^ ShuffleQueryStage (48) + +- ColumnarExchange (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner (42) + :- ^ ProjectExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : :- ^ ProjectExecTransformer (25) + : : +- ^ BroadcastHashJoinExecTransformer Inner (24) + : : :- ^ InputIteratorTransformer (6) + : : : +- ^ InputAdapter (5) + : : : +- ^ BroadcastQueryStage (4) + : : : +- ColumnarBroadcastExchange (3) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (23) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (15) + : : : :- ^ ProjectExecTransformer (8) + : : : : +- ^ Scan parquet (7) + : : : +- ^ InputIteratorTransformer (14) + : : : +- ^ InputAdapter (13) + : : : +- ^ BroadcastQueryStage (12) + : : : +- ColumnarBroadcastExchange (11) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (22) + : : +- ^ InputAdapter (21) + : : +- ^ BroadcastQueryStage (20) + : : +- ColumnarBroadcastExchange (19) + : : +- ^ ProjectExecTransformer (17) + : : +- ^ Scan parquet (16) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30) + : +- ColumnarBroadcastExchange (29) + : +- ^ ProjectExecTransformer (27) + : +- ^ Scan parquet (26) + +- ^ InputIteratorTransformer (41) + +- ^ InputAdapter (40) + +- ^ BroadcastQueryStage (39) + +- ColumnarBroadcastExchange (38) + +- ^ ProjectExecTransformer (36) + +- ^ Scan parquet (35) +- == Initial Plan == - TakeOrderedAndProject (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (75) - : : +- BroadcastHashJoin Inner BuildLeft (74) - : : :- BroadcastExchange (62) - : : : +- Filter (61) - : : : +- Scan parquet (60) - : : +- BroadcastHashJoin LeftAnti BuildRight (73) - : : :- BroadcastHashJoin LeftSemi BuildRight (68) - : : : :- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (67) - : : : +- Scan parquet (66) - : : +- BroadcastExchange (72) - : : +- Project (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- BroadcastExchange (79) - : +- Project (78) - : +- Filter (77) - : +- Scan parquet (76) - +- BroadcastExchange (85) - +- Project (84) - +- Filter (83) - +- Scan parquet (82) + TakeOrderedAndProject (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- BroadcastHashJoin Inner BuildRight (81) + :- Project (76) + : +- BroadcastHashJoin Inner BuildRight (75) + : :- Project (70) + : : +- BroadcastHashJoin Inner BuildLeft (69) + : : :- BroadcastExchange (57) + : : : +- Filter (56) + : : : +- Scan parquet (55) + : : +- BroadcastHashJoin LeftAnti BuildRight (68) + : : :- BroadcastHashJoin LeftSemi BuildRight (63) + : : : :- Project (60) + : : : : +- Filter (59) + : : : : +- Scan parquet (58) + : : : +- BroadcastExchange (62) + : : : +- Scan parquet (61) + : : +- BroadcastExchange (67) + : : +- Project (66) + : : +- Filter (65) + : : +- Scan parquet (64) + : +- BroadcastExchange (74) + : +- Project (73) + : +- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (80) + +- Project (79) + +- Filter (78) + +- Scan parquet (77) (1) Scan parquet @@ -95,406 +90,386 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) Scan parquet +(7) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(10) ProjectExecTransformer +(8) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(11) Scan parquet +(9) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(11) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(12) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) BroadcastHashJoinExecTransformer +(15) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(18) Scan parquet +(16) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(19) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) - -(20) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(21) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(19) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(20) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(24) InputAdapter +(21) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(25) InputIteratorTransformer +(22) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(26) BroadcastHashJoinExecTransformer +(23) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(27) BroadcastHashJoinExecTransformer +(24) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(28) ProjectExecTransformer +(25) ProjectExecTransformer Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(29) Scan parquet +(26) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(30) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderstatus#X] -Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) - -(31) ProjectExecTransformer +(27) ProjectExecTransformer Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(32) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [1]: [o_orderkey#X] Arguments: false -(33) ColumnarBroadcastExchange +(29) ColumnarBroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(34) BroadcastQueryStage +(30) BroadcastQueryStage Output [1]: [o_orderkey#X] Arguments: X -(35) InputAdapter +(31) InputAdapter Input [1]: [o_orderkey#X] -(36) InputIteratorTransformer +(32) InputIteratorTransformer Input [1]: [o_orderkey#X] -(37) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(38) ProjectExecTransformer +(34) ProjectExecTransformer Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(39) Scan parquet +(35) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) - -(41) ProjectExecTransformer +(36) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(43) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(44) BroadcastQueryStage +(39) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(45) InputAdapter +(40) InputAdapter Input [1]: [n_nationkey#X] -(46) InputIteratorTransformer +(41) InputIteratorTransformer Input [1]: [n_nationkey#X] -(47) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(43) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(50) ProjectExecTransformer +(45) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(51) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(47) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(48) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(49) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(50) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) TakeOrderedAndProjectExecTransformer +(53) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(59) VeloxColumnarToRowExec +(54) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(60) Scan parquet +(55) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(56) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) BroadcastExchange +(57) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(58) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(59) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) Project +(60) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(66) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(67) BroadcastExchange +(62) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(68) BroadcastHashJoin +(63) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(69) Scan parquet +(64) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(70) Filter +(65) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(71) Project +(66) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(72) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(73) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(74) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(75) Project +(70) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(76) Scan parquet +(71) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(77) Filter +(72) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(78) Project +(73) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(79) BroadcastExchange +(74) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(81) Project +(76) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(82) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(78) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(84) Project +(79) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(85) BroadcastExchange +(80) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(87) Project +(82) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(88) HashAggregate +(83) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(89) Exchange +(84) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(85) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(91) TakeOrderedAndProject +(86) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(92) AdaptiveSparkPlan +(87) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt index dff492685686..e33dc6b7fb9b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt @@ -1,40 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (25) + +- ^ SortExecTransformer (23) + +- ^ InputIteratorTransformer (22) + +- ^ InputAdapter (21) + +- ^ ShuffleQueryStage (20) + +- ColumnarExchange (19) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FlushableHashAggregateExecTransformer (10) + +- ^ ProjectExecTransformer (9) + +- ^ BroadcastHashJoinExecTransformer LeftAnti (8) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (7) + +- ^ InputAdapter (6) + +- ^ BroadcastQueryStage (5) + +- ColumnarBroadcastExchange (4) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (36) + +- Exchange (35) + +- HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin LeftAnti BuildRight (30) + :- Filter (27) + : +- Scan parquet (26) + +- BroadcastExchange (29) + +- Scan parquet (28) (1) Scan parquet @@ -44,164 +43,160 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) - -(3) Scan parquet +(2) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [1]: [o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(5) BroadcastQueryStage Output [1]: [o_custkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [o_custkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(8) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(9) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(11) FlushableHashAggregateExecTransformer +(10) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(19) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(20) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(21) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(22) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(23) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(26) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(27) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(28) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(29) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(31) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(32) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(33) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(34) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(35) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(36) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt index 950a881d1ecf..e6304542a9ce 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt @@ -1,55 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (31) + +- TakeOrderedAndProjectExecTransformer (30) + +- ^ ProjectExecTransformer (28) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ BroadcastHashJoinExecTransformer Inner (18) + :- ^ ProjectExecTransformer (10) + : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ BroadcastQueryStage (15) + +- ColumnarBroadcastExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ Scan parquet (11) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- Project (45) + +- BroadcastHashJoin Inner BuildRight (44) + :- Project (39) + : +- BroadcastHashJoin Inner BuildLeft (38) + : :- BroadcastExchange (35) + : : +- Project (34) + : : +- Filter (33) + : : +- Scan parquet (32) + : +- Filter (37) + : +- Scan parquet (36) + +- BroadcastExchange (43) + +- Project (42) + +- Filter (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,234 +56,222 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_mktsegment#X] -Arguments: ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [1]: [c_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(5) BroadcastQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(13) Scan parquet +(11) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) - -(15) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(17) ColumnarBroadcastExchange +(14) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(18) BroadcastQueryStage +(15) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(19) InputAdapter +(16) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(21) BroadcastHashJoinExecTransformer +(18) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(22) ProjectExecTransformer +(19) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(24) ProjectExecTransformer +(21) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, o_orderdate#X, o_shippriority#X, 42) AS hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(25) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(23) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(25) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(31) ProjectExecTransformer +(28) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(32) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(30) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(32) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(33) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(34) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(35) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(36) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(37) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(39) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(40) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(41) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(42) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(43) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(44) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(45) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(46) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(47) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(48) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(49) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt index 3969248bc2ec..5ea636f9ac11 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt @@ -1,46 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ ShuffleQueryStage (18) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- ^ InputAdapter (10) - +- ^ BroadcastQueryStage (9) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (10) + :- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (41) + +- Exchange (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- BroadcastHashJoin LeftSemi BuildRight (35) + :- Project (30) + : +- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -50,190 +48,182 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Arguments: ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(4) Scan parquet +(3) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(5) FilterExecTransformer -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) - -(6) ProjectExecTransformer +(4) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(7) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(8) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(9) BroadcastQueryStage +(7) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(10) InputAdapter +(8) InputAdapter Input [1]: [l_orderkey#X] -(11) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [l_orderkey#X] -(12) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(13) ProjectExecTransformer +(11) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(14) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(15) ProjectExecTransformer +(13) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(16) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(17) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(25) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(28) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(29) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(30) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(31) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(32) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(33) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(34) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(35) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(36) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(37) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(38) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(40) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(41) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt index 678a52004a3f..377110d0e7a1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt @@ -1,98 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ^ InputAdapter (61) - +- ^ ShuffleQueryStage (60) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ BroadcastQueryStage (45) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (59) + +- ^ SortExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ^ InputAdapter (55) + +- ^ ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ^ InputAdapter (49) + +- ^ ShuffleQueryStage (48) + +- ColumnarExchange (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner (42) + :- ^ ProjectExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : :- ^ ProjectExecTransformer (26) + : : +- ^ BroadcastHashJoinExecTransformer Inner (25) + : : :- ^ ProjectExecTransformer (18) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : : : : :- ^ InputIteratorTransformer (6) + : : : : : +- ^ InputAdapter (5) + : : : : : +- ^ BroadcastQueryStage (4) + : : : : : +- ColumnarBroadcastExchange (3) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (8) + : : : : +- ^ Scan parquet (7) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (24) + : : +- ^ InputAdapter (23) + : : +- ^ BroadcastQueryStage (22) + : : +- ColumnarBroadcastExchange (21) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30) + : +- ColumnarBroadcastExchange (29) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (41) + +- ^ InputAdapter (40) + +- ^ BroadcastQueryStage (39) + +- ColumnarBroadcastExchange (38) + +- ^ ProjectExecTransformer (36) + +- ^ Scan parquet (35) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (77) + : : +- BroadcastHashJoin Inner BuildRight (76) + : : :- Project (72) + : : : +- BroadcastHashJoin Inner BuildRight (71) + : : : :- Project (67) + : : : : +- BroadcastHashJoin Inner BuildLeft (66) + : : : : :- BroadcastExchange (62) + : : : : : +- Filter (61) + : : : : : +- Scan parquet (60) + : : : : +- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- BroadcastExchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (80) + : +- Filter (79) + : +- Scan parquet (78) + +- BroadcastExchange (86) + +- Project (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -102,438 +96,414 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(8) Scan parquet +(7) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(10) ProjectExecTransformer +(8) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(13) Scan parquet +(11) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) - -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(21) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(19) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(24) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarBroadcastExchange +(21) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(26) BroadcastQueryStage +(22) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(23) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(24) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) BroadcastHashJoinExecTransformer +(25) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(30) ProjectExecTransformer +(26) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) Scan parquet +(27) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(32) FilterExecTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(33) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(34) ColumnarBroadcastExchange +(29) ColumnarBroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(30) BroadcastQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(36) InputAdapter +(31) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(37) InputIteratorTransformer +(32) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(38) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(39) ProjectExecTransformer +(34) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(40) Scan parquet +(35) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(41) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) - -(42) ProjectExecTransformer +(36) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(43) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(44) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(45) BroadcastQueryStage +(39) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(46) InputAdapter +(40) InputAdapter Input [1]: [r_regionkey#X] -(47) InputIteratorTransformer +(41) InputIteratorTransformer Input [1]: [r_regionkey#X] -(48) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(49) ProjectExecTransformer +(43) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(50) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(51) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(52) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(47) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(48) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(49) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(50) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(53) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(55) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(57) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(60) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(61) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(62) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(63) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(64) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(65) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(67) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(68) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(69) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(70) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(72) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(73) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(74) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(75) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(77) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(78) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(79) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(80) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(82) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(83) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(84) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(85) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(86) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(88) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(89) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(90) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(91) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(98) Exchange +(92) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(93) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt index fa2536a365c8..cc87fe6d2bbf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt @@ -1,23 +1,22 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (18) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (11) + +- ^ RegularHashAggregateExecTransformer (9) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ ShuffleQueryStage (6) + +- ColumnarExchange (5) + +- ^ FlushableHashAggregateExecTransformer (3) + +- ^ ProjectExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (17) + +- Exchange (16) + +- HashAggregate (15) + +- Project (14) + +- Filter (13) + +- Scan parquet (12) (1) Scan parquet @@ -27,86 +26,82 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(3) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(5) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(5) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(7) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(9) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(11) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(12) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(13) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(14) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(15) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(16) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(17) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(18) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt index 19b6cc4b8aee..8716498ef067 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt @@ -1,92 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ ShuffleQueryStage (55) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- ^ InputAdapter (17) - : : : +- ^ BroadcastQueryStage (16) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- ^ InputAdapter (26) - : : +- ^ BroadcastQueryStage (25) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ BroadcastQueryStage (40) - +- ReusedExchange (39) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ^ InputAdapter (51) + +- ^ ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- ^ RegularHashAggregateExecTransformer (47) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FlushableHashAggregateExecTransformer (40) + +- ^ ProjectExecTransformer (39) + +- ^ BroadcastHashJoinExecTransformer Inner (38) + :- ^ ProjectExecTransformer (33) + : +- ^ BroadcastHashJoinExecTransformer Inner (32) + : :- ^ ProjectExecTransformer (25) + : : +- ^ BroadcastHashJoinExecTransformer Inner (24) + : : :- ^ ProjectExecTransformer (17) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (16) + : : : :- ^ ProjectExecTransformer (9) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (8) + : : : : :- ^ InputIteratorTransformer (6) + : : : : : +- ^ InputAdapter (5) + : : : : : +- ^ BroadcastQueryStage (4) + : : : : : +- ColumnarBroadcastExchange (3) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ Scan parquet (7) + : : : +- ^ InputIteratorTransformer (15) + : : : +- ^ InputAdapter (14) + : : : +- ^ BroadcastQueryStage (13) + : : : +- ColumnarBroadcastExchange (12) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ BroadcastQueryStage (21) + : : +- ColumnarBroadcastExchange (20) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ BroadcastQueryStage (29) + : +- ColumnarBroadcastExchange (28) + : +- ^ Scan parquet (26) + +- ^ InputIteratorTransformer (37) + +- ^ InputAdapter (36) + +- ^ BroadcastQueryStage (35) + +- ReusedExchange (34) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (87) + +- Exchange (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- BroadcastHashJoin Inner BuildRight (81) + :- Project (77) + : +- BroadcastHashJoin Inner BuildRight (76) + : :- Project (72) + : : +- BroadcastHashJoin Inner BuildRight (71) + : : :- Project (67) + : : : +- BroadcastHashJoin Inner BuildRight (66) + : : : :- Project (62) + : : : : +- BroadcastHashJoin Inner BuildLeft (61) + : : : : :- BroadcastExchange (58) + : : : : : +- Filter (57) + : : : : : +- Scan parquet (56) + : : : : +- Filter (60) + : : : : +- Scan parquet (59) + : : : +- BroadcastExchange (65) + : : : +- Filter (64) + : : : +- Scan parquet (63) + : : +- BroadcastExchange (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- BroadcastExchange (75) + : +- Filter (74) + : +- Scan parquet (73) + +- BroadcastExchange (80) + +- Filter (79) + +- Scan parquet (78) (1) Scan parquet @@ -96,406 +91,386 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) Scan parquet +(7) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(10) BroadcastHashJoinExecTransformer +(8) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) Scan parquet +(10) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(13) FilterExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) - -(14) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(15) ColumnarBroadcastExchange +(12) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(16) BroadcastQueryStage +(13) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(17) InputAdapter +(14) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(18) InputIteratorTransformer +(15) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(19) BroadcastHashJoinExecTransformer +(16) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(20) ProjectExecTransformer +(17) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(21) Scan parquet +(18) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(22) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(23) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(24) ColumnarBroadcastExchange +(20) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(25) BroadcastQueryStage +(21) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(26) InputAdapter +(22) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(27) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(28) BroadcastHashJoinExecTransformer +(24) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(29) ProjectExecTransformer +(25) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(30) Scan parquet +(26) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(31) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) - -(32) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(33) ColumnarBroadcastExchange +(28) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(34) BroadcastQueryStage +(29) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(35) InputAdapter +(30) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(36) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(37) BroadcastHashJoinExecTransformer +(32) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(38) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(39) ReusedExchange [Reuses operator id: 33] +(34) ReusedExchange [Reuses operator id: 28] Output [2]: [n_nationkey#X, n_name#X] -(40) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(41) InputAdapter +(36) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(42) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(43) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(44) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(45) FlushableHashAggregateExecTransformer +(40) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(45) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(47) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(49) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(50) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(51) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(52) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(53) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(56) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(57) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(58) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(59) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(60) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(61) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(62) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(63) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(64) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(65) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(67) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(72) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(73) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(74) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(75) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(77) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(80) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(82) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(83) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(84) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(85) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(86) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(87) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt index 3aa6e39645cc..94e003e55d93 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt @@ -1,125 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (121) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ BroadcastQueryStage (6) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- ^ InputAdapter (18) - : : : : : +- ^ BroadcastQueryStage (17) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- ^ InputAdapter (27) - : : : : +- ^ BroadcastQueryStage (26) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- ^ InputAdapter (36) - : : : +- ^ BroadcastQueryStage (35) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ BroadcastQueryStage (44) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- ^ InputAdapter (54) - : +- ^ BroadcastQueryStage (53) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- ^ InputAdapter (64) - +- ^ BroadcastQueryStage (63) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (76) + +- ^ SortExecTransformer (74) + +- ^ InputIteratorTransformer (73) + +- ^ InputAdapter (72) + +- ^ ShuffleQueryStage (71) + +- ColumnarExchange (70) + +- ^ ProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ InputIteratorTransformer (66) + +- ^ InputAdapter (65) + +- ^ ShuffleQueryStage (64) + +- ColumnarExchange (63) + +- ^ ProjectExecTransformer (61) + +- ^ FlushableHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ BroadcastHashJoinExecTransformer Inner (58) + :- ^ ProjectExecTransformer (50) + : +- ^ BroadcastHashJoinExecTransformer Inner (49) + : :- ^ ProjectExecTransformer (42) + : : +- ^ BroadcastHashJoinExecTransformer Inner (41) + : : :- ^ ProjectExecTransformer (34) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : : : :- ^ ProjectExecTransformer (26) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (25) + : : : : :- ^ ProjectExecTransformer (18) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) + : : : : : :- ^ ProjectExecTransformer (10) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : : : : : : :- ^ InputIteratorTransformer (7) + : : : : : : : +- ^ InputAdapter (6) + : : : : : : : +- ^ BroadcastQueryStage (5) + : : : : : : : +- ColumnarBroadcastExchange (4) + : : : : : : : +- ^ ProjectExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ Scan parquet (8) + : : : : : +- ^ InputIteratorTransformer (16) + : : : : : +- ^ InputAdapter (15) + : : : : : +- ^ BroadcastQueryStage (14) + : : : : : +- ColumnarBroadcastExchange (13) + : : : : : +- ^ Scan parquet (11) + : : : : +- ^ InputIteratorTransformer (24) + : : : : +- ^ InputAdapter (23) + : : : : +- ^ BroadcastQueryStage (22) + : : : : +- ColumnarBroadcastExchange (21) + : : : : +- ^ Scan parquet (19) + : : : +- ^ InputIteratorTransformer (32) + : : : +- ^ InputAdapter (31) + : : : +- ^ BroadcastQueryStage (30) + : : : +- ColumnarBroadcastExchange (29) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (40) + : : +- ^ InputAdapter (39) + : : +- ^ BroadcastQueryStage (38) + : : +- ColumnarBroadcastExchange (37) + : : +- ^ Scan parquet (35) + : +- ^ InputIteratorTransformer (48) + : +- ^ InputAdapter (47) + : +- ^ BroadcastQueryStage (46) + : +- ColumnarBroadcastExchange (45) + : +- ^ Scan parquet (43) + +- ^ InputIteratorTransformer (57) + +- ^ InputAdapter (56) + +- ^ BroadcastQueryStage (55) + +- ColumnarBroadcastExchange (54) + +- ^ ProjectExecTransformer (52) + +- ^ Scan parquet (51) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (120) + +- Exchange (119) + +- HashAggregate (118) + +- Exchange (117) + +- HashAggregate (116) + +- Project (115) + +- BroadcastHashJoin Inner BuildRight (114) + :- Project (109) + : +- BroadcastHashJoin Inner BuildRight (108) + : :- Project (104) + : : +- BroadcastHashJoin Inner BuildRight (103) + : : :- Project (99) + : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : :- Project (94) + : : : : +- BroadcastHashJoin Inner BuildRight (93) + : : : : :- Project (89) + : : : : : +- BroadcastHashJoin Inner BuildRight (88) + : : : : : :- Project (84) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (83) + : : : : : : :- BroadcastExchange (80) + : : : : : : : +- Project (79) + : : : : : : : +- Filter (78) + : : : : : : : +- Scan parquet (77) + : : : : : : +- Filter (82) + : : : : : : +- Scan parquet (81) + : : : : : +- BroadcastExchange (87) + : : : : : +- Filter (86) + : : : : : +- Scan parquet (85) + : : : : +- BroadcastExchange (92) + : : : : +- Filter (91) + : : : : +- Scan parquet (90) + : : : +- BroadcastExchange (97) + : : : +- Filter (96) + : : : +- Scan parquet (95) + : : +- BroadcastExchange (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- BroadcastExchange (107) + : +- Filter (106) + : +- Scan parquet (105) + +- BroadcastExchange (113) + +- Project (112) + +- Filter (111) + +- Scan parquet (110) (1) Scan parquet @@ -129,566 +121,534 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(5) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(13) Scan parquet +(11) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(21) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(22) Scan parquet +(19) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(23) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) - -(24) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(25) ColumnarBroadcastExchange +(21) ColumnarBroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(26) BroadcastQueryStage +(22) BroadcastQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(27) InputAdapter +(23) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(28) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(29) BroadcastHashJoinExecTransformer +(25) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(26) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(31) Scan parquet +(27) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(32) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(33) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(34) ColumnarBroadcastExchange +(29) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(30) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(36) InputAdapter +(31) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(37) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(38) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(39) ProjectExecTransformer +(34) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(40) Scan parquet +(35) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(41) FilterExecTransformer -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(42) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: false -(43) ColumnarBroadcastExchange +(37) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastQueryStage +(38) BroadcastQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(45) InputAdapter +(39) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(46) InputIteratorTransformer +(40) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(47) BroadcastHashJoinExecTransformer +(41) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(42) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(49) Scan parquet +(43) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(50) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(51) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(52) ColumnarBroadcastExchange +(45) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(53) BroadcastQueryStage +(46) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(54) InputAdapter +(47) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(55) InputIteratorTransformer +(48) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(56) BroadcastHashJoinExecTransformer +(49) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(57) ProjectExecTransformer +(50) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(58) Scan parquet +(51) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(59) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) - -(60) ProjectExecTransformer +(52) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(61) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(62) ColumnarBroadcastExchange +(54) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) BroadcastQueryStage +(55) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(64) InputAdapter +(56) InputAdapter Input [1]: [r_regionkey#X] -(65) InputIteratorTransformer +(57) InputIteratorTransformer Input [1]: [r_regionkey#X] -(66) BroadcastHashJoinExecTransformer +(58) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(67) ProjectExecTransformer +(59) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(68) FlushableHashAggregateExecTransformer +(60) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(69) ProjectExecTransformer +(61) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(70) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(63) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(64) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(65) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(66) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(68) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(69) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(70) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(71) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(72) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(73) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(74) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(76) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(77) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(78) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(79) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(80) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(81) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(82) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(84) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(85) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(86) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(87) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(89) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(90) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(91) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(92) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(94) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(95) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(96) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(97) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(99) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(100) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(101) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(102) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(105) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(106) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(107) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(110) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(111) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(112) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(113) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(114) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(115) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(116) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(117) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(118) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(127) Exchange +(119) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(120) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(121) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt index b00947a3e3e0..d337c7f9e468 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt @@ -1,96 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ^ InputAdapter (60) - +- ^ ShuffleQueryStage (59) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ BroadcastQueryStage (6) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (58) + +- ^ SortExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : :- ^ ProjectExecTransformer (26) + : : +- ^ BroadcastHashJoinExecTransformer Inner (25) + : : :- ^ ProjectExecTransformer (18) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ ProjectExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (24) + : : +- ^ InputAdapter (23) + : : +- ^ BroadcastQueryStage (22) + : : +- ColumnarBroadcastExchange (21) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30) + : +- ColumnarBroadcastExchange (29) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38) + +- ColumnarBroadcastExchange (37) + +- ^ Scan parquet (35) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (91) + +- Exchange (90) + +- HashAggregate (89) + +- Exchange (88) + +- HashAggregate (87) + +- Project (86) + +- BroadcastHashJoin Inner BuildRight (85) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (76) + : : +- BroadcastHashJoin Inner BuildRight (75) + : : :- Project (71) + : : : +- BroadcastHashJoin Inner BuildRight (70) + : : : :- Project (66) + : : : : +- BroadcastHashJoin Inner BuildLeft (65) + : : : : :- BroadcastExchange (62) + : : : : : +- Project (61) + : : : : : +- Filter (60) + : : : : : +- Scan parquet (59) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- BroadcastExchange (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- BroadcastExchange (79) + : +- Filter (78) + : +- Scan parquet (77) + +- BroadcastExchange (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -100,430 +94,406 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(5) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(8) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(13) Scan parquet +(11) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(21) ProjectExecTransformer +(18) ProjectExecTransformer Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(22) Scan parquet +(19) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(23) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) - -(24) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(25) ColumnarBroadcastExchange +(21) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(26) BroadcastQueryStage +(22) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(27) InputAdapter +(23) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(28) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(29) BroadcastHashJoinExecTransformer +(25) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(30) ProjectExecTransformer +(26) ProjectExecTransformer Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(31) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(32) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: isnotnull(o_orderkey#X) - -(33) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: false -(34) ColumnarBroadcastExchange +(29) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(30) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(36) InputAdapter +(31) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(37) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(38) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(39) ProjectExecTransformer +(34) ProjectExecTransformer Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(40) Scan parquet +(35) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(41) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(42) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarBroadcastExchange +(37) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastQueryStage +(38) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(39) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(40) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) BroadcastHashJoinExecTransformer +(41) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(42) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(44) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(46) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(47) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(48) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(49) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(52) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(53) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(54) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(55) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(56) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(59) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(60) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(61) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(62) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(63) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(64) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(65) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(66) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(67) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(68) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(69) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(70) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(71) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(72) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(73) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(74) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(76) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(77) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(78) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(79) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(81) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(84) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(85) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(86) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(87) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(88) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(89) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(90) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(91) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt index 0cacd70fdca7..c5d58658acd5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt @@ -1,31 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (27) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (18) + +- ^ SortExecTransformer (16) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (3) + +- ^ ProjectExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (26) + +- Exchange (25) + +- HashAggregate (24) + +- Exchange (23) + +- HashAggregate (22) + +- Project (21) + +- Filter (20) + +- Scan parquet (19) (1) Scan parquet @@ -35,120 +34,116 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) FilterExecTransformer -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Arguments: (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(3) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) ProjectExecTransformer +(4) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(6) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(6) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(7) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(8) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(9) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(12) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(13) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(14) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(15) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(16) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(18) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(19) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(20) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(21) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(22) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(23) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(24) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(25) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(26) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(27) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt index 2977896f3da8..c599bb00a7bb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt @@ -1,68 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (39) + +- TakeOrderedAndProjectExecTransformer (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- ^ ProjectExecTransformer (29) + +- ^ FlushableHashAggregateExecTransformer (28) + +- ^ ProjectExecTransformer (27) + +- ^ BroadcastHashJoinExecTransformer Inner (26) + :- ^ ProjectExecTransformer (19) + : +- ^ BroadcastHashJoinExecTransformer Inner (18) + : :- ^ ProjectExecTransformer (10) + : : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : : :- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ Scan parquet (2) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ BroadcastQueryStage (15), Statistics(X) + : +- ColumnarBroadcastExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ Scan parquet (11) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ BroadcastQueryStage (23), Statistics(X) + +- ColumnarBroadcastExchange (22) + +- ^ Scan parquet (20) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- BroadcastHashJoin Inner BuildRight (57) + :- Project (53) + : +- BroadcastHashJoin Inner BuildRight (52) + : :- Project (47) + : : +- BroadcastHashJoin Inner BuildRight (46) + : : :- Filter (41) + : : : +- Scan parquet (40) + : : +- BroadcastExchange (45) + : : +- Project (44) + : : +- Filter (43) + : : +- Scan parquet (42) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- BroadcastExchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,296 +68,280 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) Scan parquet +(2) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(4) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(5) ProjectExecTransformer +(3) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(6) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(7) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(8) BroadcastQueryStage +(6) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(9) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(10) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(13) Scan parquet +(11) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) - -(15) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(16) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(17) ColumnarBroadcastExchange +(14) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(18) BroadcastQueryStage +(15) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(19) InputAdapter +(16) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(21) BroadcastHashJoinExecTransformer +(18) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(22) ProjectExecTransformer +(19) ProjectExecTransformer Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) Scan parquet +(20) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(24) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(25) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(26) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(27) BroadcastQueryStage +(23) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(28) InputAdapter +(24) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(29) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(30) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(31) ProjectExecTransformer +(27) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(28) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(33) ProjectExecTransformer +(29) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(34) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(31) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(32) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(33) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(34) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(40) ProjectExecTransformer +(36) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(41) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(38) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(40) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(41) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(45) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(47) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(48) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(49) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(50) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(51) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(53) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(55) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(56) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(59) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(60) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(61) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt index 2a860ed5c063..459c52738acb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt @@ -1,59 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ^ InputAdapter (34) - +- ^ ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- ^ InputAdapter (8) - : +- ^ BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ FilterExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ BroadcastHashJoinExecTransformer Inner (17) + :- ^ ProjectExecTransformer (9) + : +- ^ BroadcastHashJoinExecTransformer Inner (8) + : :- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ BroadcastQueryStage (14), Statistics(X) + +- ColumnarBroadcastExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (54) + +- Exchange (53) + +- Filter (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- Project (42) + : +- BroadcastHashJoin Inner BuildRight (41) + : :- Filter (37) + : : +- Scan parquet (36) + : +- BroadcastExchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- BroadcastExchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -63,481 +60,464 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(3) Scan parquet +(2) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(4) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(5) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(7) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(8) InputAdapter +(6) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(9) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(10) BroadcastHashJoinExecTransformer +(8) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(12) Scan parquet +(10) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(13) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) - -(14) ProjectExecTransformer +(11) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [1]: [n_nationkey#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [n_nationkey#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(21) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(22) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(23) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(24) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(22) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(24) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(30) FilterExecTransformer +(27) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(29) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(31) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(33) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(36) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(37) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(38) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(39) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(40) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(42) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(43) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(44) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(45) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(46) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(48) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(49) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(50) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(51) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(55) Filter +(52) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(53) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(54) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +Subquery:1 Hosting operator id = 27 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner (65) - : :- ^ FilterExecTransformer (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- ^ InputAdapter (63) - : +- ^ BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ^ InputAdapter (73) + +- ^ ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ FlushableHashAggregateExecTransformer (69) + +- ^ ProjectExecTransformer (68) + +- ^ BroadcastHashJoinExecTransformer Inner (67) + :- ^ ProjectExecTransformer (62) + : +- ^ BroadcastHashJoinExecTransformer Inner (61) + : :- ^ Scan parquet (56) + : +- ^ InputIteratorTransformer (60) + : +- ^ InputAdapter (59) + : +- ^ BroadcastQueryStage (58), Statistics(X) + : +- ReusedExchange (57) + +- ^ InputIteratorTransformer (66) + +- ^ InputAdapter (65) + +- ^ BroadcastQueryStage (64), Statistics(X) + +- ReusedExchange (63) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Project (91) + +- BroadcastHashJoin Inner BuildRight (90) + :- Project (85) + : +- BroadcastHashJoin Inner BuildRight (84) + : :- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (83) + : +- Filter (82) + : +- Scan parquet (81) + +- BroadcastExchange (89) + +- Project (88) + +- Filter (87) + +- Scan parquet (86) + + +(56) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) FilterExecTransformer -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(61) ReusedExchange [Reuses operator id: 6] +(57) ReusedExchange [Reuses operator id: 4] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(58) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(59) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(60) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(61) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(66) ProjectExecTransformer +(62) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(63) ReusedExchange [Reuses operator id: 13] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(64) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(65) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(66) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(67) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(69) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(70) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(71) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(72) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(73) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(74) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(75) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(80) ProjectExecTransformer +(76) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(81) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(78) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(79) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(80) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(89) Project +(85) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(86) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(87) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(88) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(89) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(90) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(95) Project +(91) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(92) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(93) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(94) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt index 99576f6e063a..d0dca8b40c03 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt @@ -1,44 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (26) + +- ^ SortExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer Inner (9) + :- ^ InputIteratorTransformer (6) + : +- ^ InputAdapter (5) + : +- ^ BroadcastQueryStage (4), Statistics(X) + : +- ColumnarBroadcastExchange (3) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (8) + +- ^ Scan parquet (7) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin Inner BuildLeft (33) + :- BroadcastExchange (29) + : +- Filter (28) + : +- Scan parquet (27) + +- Project (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -48,182 +46,174 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: isnotnull(o_orderkey#X) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(7) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) - -(10) ProjectExecTransformer +(8) ProjectExecTransformer Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(13) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(14) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(15) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(16) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(24) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(28) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(29) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(30) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(31) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(32) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(34) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(35) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(36) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(37) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(38) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(39) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt index 8ebeb7d32d52..a9269637304d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt @@ -1,53 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (51) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftOuter (9) :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ ProjectExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (50) + +- Exchange (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- Exchange (44) + +- HashAggregate (43) + +- Project (42) + +- BroadcastHashJoin LeftOuter BuildRight (41) + :- Scan parquet (36) + +- BroadcastExchange (40) + +- Project (39) + +- Filter (38) + +- Scan parquet (37) (1) Scan parquet @@ -63,224 +62,220 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(3) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) - -(4) ProjectExecTransformer +(3) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(5) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(7) BroadcastQueryStage +(6) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(8) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(9) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(10) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(11) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(12) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(13) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, count#X] Input [2]: [c_custkey#X, count#X] -(14) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(14) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(16) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(19) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(21) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(25) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(29) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(31) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(33) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(36) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(37) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(38) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(39) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(40) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(42) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(43) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(44) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(46) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(47) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(48) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(49) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(50) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(51) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt index bfba707a5792..2b4a4b697d1d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt @@ -1,37 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (32) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (20) + +- ^ ProjectExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer Inner (9) + :- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (31) + +- Exchange (30) + +- HashAggregate (29) + +- Project (28) + +- BroadcastHashJoin Inner BuildRight (27) + :- Project (23) + : +- Filter (22) + : +- Scan parquet (21) + +- BroadcastExchange (26) + +- Filter (25) + +- Scan parquet (24) (1) Scan parquet @@ -41,152 +39,144 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) Scan parquet +(3) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(5) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: isnotnull(p_partkey#X) - -(6) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [p_partkey#X, p_type#X] Arguments: false -(7) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(8) BroadcastQueryStage +(6) BroadcastQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(9) InputAdapter +(7) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(10) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(13) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(15) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(21) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(22) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(23) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(24) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(25) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(26) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(27) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(28) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(29) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(30) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(31) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(32) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt index 06b833361548..3bb88d987670 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt @@ -1,44 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (24) + +- AQEShuffleRead (23) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ BroadcastHashJoinExecTransformer Inner (18) + :- ^ InputIteratorTransformer (6) + : +- ^ InputAdapter (5) + : +- ^ BroadcastQueryStage (4), Statistics(X) + : +- ColumnarBroadcastExchange (3) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (17) + +- ^ RegularHashAggregateExecTransformer (16) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ FlushableHashAggregateExecTransformer (9) + +- ^ ProjectExecTransformer (8) + +- ^ Scan parquet (7) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (38) + +- Exchange (37) + +- Project (36) + +- BroadcastHashJoin Inner BuildLeft (35) + :- BroadcastExchange (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Filter (34) + +- HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- Filter (29) + +- Scan parquet (28) (1) Scan parquet @@ -48,337 +46,324 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: isnotnull(s_suppkey#X) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) Scan parquet +(7) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) - -(10) ProjectExecTransformer +(8) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) FlushableHashAggregateExecTransformer +(9) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(12) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(13) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(14) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(15) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(16) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(19) FilterExecTransformer +(17) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(18) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(23) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(25) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(26) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(27) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(28) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(29) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(30) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(31) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(32) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(33) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(36) Filter +(34) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(35) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(38) Project +(36) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(37) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(38) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +Subquery:1 Hosting operator id = 17 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ FilterExecTransformer (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ^ InputAdapter (47) + +- ^ ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ ProjectExecTransformer (43) + +- ^ FlushableHashAggregateExecTransformer (42) + +- ^ ProjectExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Exchange (59) + +- HashAggregate (58) + +- Project (57) + +- Filter (56) + +- Scan parquet (55) -(42) Scan parquet +(40) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) - -(44) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(42) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(43) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(45) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(46) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(47) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(48) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(53) ProjectExecTransformer +(50) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(54) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(55) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(56) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(57) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(58) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(59) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(60) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(64) HashAggregate +(61) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(62) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt index af623a3a2f68..2d1099920a41 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt @@ -1,57 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (62) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (41) + +- ^ SortExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ RegularHashAggregateExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ ProjectExecTransformer (27) + +- ^ FlushableHashAggregateExecTransformer (26) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ BroadcastHashJoinExecTransformer Inner (16) + :- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + : :- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ BroadcastQueryStage (6), Statistics(X) + : +- ColumnarBroadcastExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ BroadcastQueryStage (13), Statistics(X) + +- ColumnarBroadcastExchange (12) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (61) + +- Exchange (60) + +- HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- BroadcastHashJoin LeftAnti BuildRight (48) + : :- Filter (43) + : : +- Scan parquet (42) + : +- BroadcastExchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- BroadcastExchange (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -61,252 +66,278 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: isnotnull(ps_partkey#X) +(2) Scan parquet +Output [2]: [s_suppkey#X, s_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_comment)] +ReadSchema: struct + +(3) ProjectExecTransformer +Output [1]: [s_suppkey#X] +Input [2]: [s_suppkey#X, s_comment#X] + +(4) WholeStageCodegenTransformer (X) +Input [1]: [s_suppkey#X] +Arguments: false + +(5) ColumnarBroadcastExchange +Input [1]: [s_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] + +(6) BroadcastQueryStage +Output [1]: [s_suppkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [s_suppkey#X] -(3) Scan parquet +(8) InputIteratorTransformer +Input [1]: [s_suppkey#X] + +(9) BroadcastHashJoinExecTransformer +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join condition: None + +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(4) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) - -(5) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(6) ColumnarBroadcastExchange +(12) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(7) BroadcastQueryStage +(13) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(8) InputAdapter +(14) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(9) InputIteratorTransformer +(15) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(10) BroadcastHashJoinExecTransformer +(16) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(11) ProjectExecTransformer +(17) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(13) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(14) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(21) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(22) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(23) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(24) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(26) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(27) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(29) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(30) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(31) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(32) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(33) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(35) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(37) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(39) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(41) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(42) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(43) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(45) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(46) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(47) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(49) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(50) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(51) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(53) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(54) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(57) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(58) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(59) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(61) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(62) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt index cc13e5f3b886..48f5a91500b2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt @@ -1,86 +1,83 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (83) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (50) + +- TakeOrderedAndProjectExecTransformer (49) + +- ^ RegularHashAggregateExecTransformer (47) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FlushableHashAggregateExecTransformer (40) + +- ^ ProjectExecTransformer (39) + +- ^ BroadcastHashJoinExecTransformer Inner (38) + :- ^ ProjectExecTransformer (26) + : +- ^ BroadcastHashJoinExecTransformer Inner (25) + : :- ^ InputIteratorTransformer (6) + : : +- ^ InputAdapter (5) + : : +- ^ BroadcastQueryStage (4), Statistics(X) + : : +- ColumnarBroadcastExchange (3) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (24) + : :- ^ Scan parquet (7) + : +- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ BroadcastQueryStage (21), Statistics(X) + : +- ColumnarBroadcastExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ FilterExecTransformer (17) + : +- ^ RegularHashAggregateExecTransformer (16) + : +- ^ InputIteratorTransformer (15) + : +- ^ InputAdapter (14) + : +- ^ ShuffleQueryStage (13), Statistics(X) + : +- ColumnarExchange (12) + : +- ^ ProjectExecTransformer (10) + : +- ^ FlushableHashAggregateExecTransformer (9) + : +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (37) + +- ^ InputAdapter (36) + +- ^ BroadcastQueryStage (35), Statistics(X) + +- ColumnarBroadcastExchange (34) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (32) + :- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ BroadcastQueryStage (29), Statistics(X) + +- ReusedExchange (28) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (82) + +- HashAggregate (81) + +- Exchange (80) + +- HashAggregate (79) + +- Project (78) + +- BroadcastHashJoin Inner BuildRight (77) + :- Project (65) + : +- BroadcastHashJoin Inner BuildLeft (64) + : :- BroadcastExchange (53) + : : +- Filter (52) + : : +- Scan parquet (51) + : +- BroadcastHashJoin LeftSemi BuildRight (63) + : :- Filter (55) + : : +- Scan parquet (54) + : +- BroadcastExchange (62) + : +- Project (61) + : +- Filter (60) + : +- HashAggregate (59) + : +- Exchange (58) + : +- HashAggregate (57) + : +- Scan parquet (56) + +- BroadcastExchange (76) + +- BroadcastHashJoin LeftSemi BuildRight (75) + :- Filter (67) + : +- Scan parquet (66) + +- BroadcastExchange (74) + +- Project (73) + +- Filter (72) + +- HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Scan parquet (68) (1) Scan parquet @@ -90,387 +87,375 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_name#X] -Arguments: isnotnull(c_custkey#X) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_name#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(8) Scan parquet +(7) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) - -(10) Scan parquet +(8) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(9) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(13) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(12) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(13) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(14) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(15) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(16) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(17) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(20) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(21) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(24) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(25) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(26) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(27) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) FilterExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: isnotnull(l_orderkey#X) - -(31) ReusedExchange [Reuses operator id: 22] +(28) ReusedExchange [Reuses operator id: 20] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(29) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(32) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(36) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(40) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(43) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(44) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(45) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(46) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(47) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(49) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(51) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(52) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(53) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(54) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(55) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(56) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(57) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(58) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(59) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(60) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(61) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(62) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(63) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(64) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(65) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(67) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(68) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(69) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(70) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(71) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(72) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(73) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(74) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(76) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(78) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(79) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(80) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(81) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(82) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(83) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt index 3350a6c3009f..603998ea792a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt @@ -1,36 +1,34 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (31) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (19) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer Inner (9) + :- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (30) + +- Exchange (29) + +- HashAggregate (28) + +- Project (27) + +- BroadcastHashJoin Inner BuildRight (26) + :- Project (22) + : +- Filter (21) + : +- Scan parquet (20) + +- BroadcastExchange (25) + +- Filter (24) + +- Scan parquet (23) (1) Scan parquet @@ -40,148 +38,140 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) FilterExecTransformer -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Arguments: ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(4) Scan parquet +(3) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(5) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) - -(6) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(7) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(8) BroadcastQueryStage +(6) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(9) InputAdapter +(7) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(10) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(13) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(15) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(20) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(21) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(22) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(23) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(24) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(25) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(26) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(27) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(28) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(29) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(30) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(31) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt index a95115d00b26..c0c669bbce45 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt @@ -1,103 +1,101 @@ == Physical Plan == -AdaptiveSparkPlan (104) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (67) - +- AQEShuffleRead (66) - +- ShuffleQueryStage (65), Statistics(X) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (51) - : :- ^ InputIteratorTransformer (9) - : : +- ^ InputAdapter (8) - : : +- ^ AQEShuffleRead (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- ^ InputAdapter (49) - : +- ^ BroadcastQueryStage (48), Statistics(X) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner (44) - : :- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (20) - : : :- ^ FilterExecTransformer (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- ^ InputAdapter (18) - : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ FilterExecTransformer (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ^ InputAdapter (39) - : +- ^ ShuffleQueryStage (38), Statistics(X) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ FilterExecTransformer (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- ^ InputAdapter (31) - : +- ^ BroadcastQueryStage (30), Statistics(X) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- ^ InputAdapter (59) - +- ^ BroadcastQueryStage (58), Statistics(X) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ FilterExecTransformer (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (65) + +- AQEShuffleRead (64) + +- ShuffleQueryStage (63), Statistics(X) + +- ColumnarExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ BroadcastHashJoinExecTransformer Inner (59) + :- ^ ProjectExecTransformer (51) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (50) + : :- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ AQEShuffleRead (6) + : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (49) + : +- ^ InputAdapter (48) + : +- ^ BroadcastQueryStage (47), Statistics(X) + : +- ColumnarBroadcastExchange (46) + : +- AQEShuffleRead (45) + : +- ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- ^ ProjectExecTransformer (41) + : +- ^ BroadcastHashJoinExecTransformer Inner (40) + : :- ^ InputIteratorTransformer (22) + : : +- ^ InputAdapter (21) + : : +- ^ BroadcastQueryStage (20), Statistics(X) + : : +- ColumnarBroadcastExchange (19) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (17) + : : :- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (16) + : : +- ^ InputAdapter (15) + : : +- ^ BroadcastQueryStage (14), Statistics(X) + : : +- ColumnarBroadcastExchange (13) + : : +- ^ ProjectExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ FilterExecTransformer (39) + : +- ^ ProjectExecTransformer (38) + : +- ^ RegularHashAggregateExecTransformer (37) + : +- ^ InputIteratorTransformer (36) + : +- ^ InputAdapter (35) + : +- ^ ShuffleQueryStage (34), Statistics(X) + : +- ColumnarExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FlushableHashAggregateExecTransformer (30) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (29) + : :- ^ ProjectExecTransformer (24) + : : +- ^ Scan parquet (23) + : +- ^ InputIteratorTransformer (28) + : +- ^ InputAdapter (27) + : +- ^ BroadcastQueryStage (26), Statistics(X) + : +- ReusedExchange (25) + +- ^ InputIteratorTransformer (58) + +- ^ InputAdapter (57) + +- ^ BroadcastQueryStage (56), Statistics(X) + +- ColumnarBroadcastExchange (55) + +- ^ ProjectExecTransformer (53) + +- ^ Scan parquet (52) +- == Initial Plan == - Sort (103) - +- Exchange (102) - +- Project (101) - +- BroadcastHashJoin Inner BuildRight (100) - :- Project (95) - : +- ShuffledHashJoin LeftSemi BuildRight (94) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Exchange (93) - : +- Project (92) - : +- BroadcastHashJoin Inner BuildLeft (91) - : :- BroadcastExchange (78) - : : +- BroadcastHashJoin LeftSemi BuildRight (77) - : : :- Filter (72) - : : : +- Scan parquet (71) - : : +- BroadcastExchange (76) - : : +- Project (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Filter (90) - : +- HashAggregate (89) - : +- Exchange (88) - : +- HashAggregate (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Project (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (85) - : +- Project (84) - : +- Filter (83) - : +- Scan parquet (82) - +- BroadcastExchange (99) - +- Project (98) - +- Filter (97) - +- Scan parquet (96) + Sort (101) + +- Exchange (100) + +- Project (99) + +- BroadcastHashJoin Inner BuildRight (98) + :- Project (93) + : +- ShuffledHashJoin LeftSemi BuildRight (92) + : :- Exchange (68) + : : +- Filter (67) + : : +- Scan parquet (66) + : +- Exchange (91) + : +- Project (90) + : +- BroadcastHashJoin Inner BuildLeft (89) + : :- BroadcastExchange (76) + : : +- BroadcastHashJoin LeftSemi BuildRight (75) + : : :- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (74) + : : +- Project (73) + : : +- Filter (72) + : : +- Scan parquet (71) + : +- Filter (88) + : +- HashAggregate (87) + : +- Exchange (86) + : +- HashAggregate (85) + : +- BroadcastHashJoin LeftSemi BuildRight (84) + : :- Project (79) + : : +- Filter (78) + : : +- Scan parquet (77) + : +- BroadcastExchange (83) + : +- Project (82) + : +- Filter (81) + : +- Scan parquet (80) + +- BroadcastExchange (97) + +- Project (96) + +- Filter (95) + +- Scan parquet (94) (1) Scan parquet @@ -107,450 +105,442 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: isnotnull(s_nationkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(6) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(7) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(9) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) - -(12) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) - -(14) ProjectExecTransformer +(11) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(19) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(20) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(21) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(22) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(23) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) FilterExecTransformer -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) - -(28) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(25) ReusedExchange [Reuses operator id: 13] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(26) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(27) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(28) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(29) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(30) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(33) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(34) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(35) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(36) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(37) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(39) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(40) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer -Output [1]: [ps_suppkey#X] +(41) ProjectExecTransformer +Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) -Input [1]: [ps_suppkey#X] +(42) WholeStageCodegenTransformer (X) +Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(43) ColumnarExchange +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] + +(44) ShuffleQueryStage +Output [1]: [ps_suppkey#X] +Arguments: X + +(45) AQEShuffleRead +Input [1]: [ps_suppkey#X] +Arguments: local + +(46) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(47) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(48) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(49) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(50) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(51) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(52) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) - -(55) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(55) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(56) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(57) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(58) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(59) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(60) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(61) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(62) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(63) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) AQEShuffleRead +(64) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(67) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(68) Scan parquet +(66) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(69) Filter +(67) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(70) Exchange +(68) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(69) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(72) Filter +(70) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(73) Scan parquet +(71) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(74) Filter +(72) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(75) Project +(73) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(76) BroadcastExchange +(74) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(77) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(78) BroadcastExchange +(76) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(79) Scan parquet +(77) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(80) Filter +(78) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(81) Project +(79) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(82) Scan parquet +(80) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(83) Filter +(81) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(84) Project +(82) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(85) BroadcastExchange +(83) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(87) HashAggregate +(85) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(88) Exchange +(86) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(87) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(90) Filter +(88) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(91) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(92) Project +(90) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Exchange +(91) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(95) Project +(93) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(96) Scan parquet +(94) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(97) Filter +(95) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(98) Project +(96) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(99) BroadcastExchange +(97) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(100) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(101) Project +(99) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(102) Exchange +(100) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Sort +(101) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(104) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt index 2e1ddb4e0851..f8a97759d567 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt @@ -1,90 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (86) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ^ InputAdapter (49) + +- ^ ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner (42) + :- ^ ProjectExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : :- ^ ProjectExecTransformer (25) + : : +- ^ BroadcastHashJoinExecTransformer Inner (24) + : : :- ^ InputIteratorTransformer (6) + : : : +- ^ InputAdapter (5) + : : : +- ^ BroadcastQueryStage (4), Statistics(X) + : : : +- ColumnarBroadcastExchange (3) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (23) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (15) + : : : :- ^ ProjectExecTransformer (8) + : : : : +- ^ Scan parquet (7) + : : : +- ^ InputIteratorTransformer (14) + : : : +- ^ InputAdapter (13) + : : : +- ^ BroadcastQueryStage (12), Statistics(X) + : : : +- ColumnarBroadcastExchange (11) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (22) + : : +- ^ InputAdapter (21) + : : +- ^ BroadcastQueryStage (20), Statistics(X) + : : +- ColumnarBroadcastExchange (19) + : : +- ^ ProjectExecTransformer (17) + : : +- ^ Scan parquet (16) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30), Statistics(X) + : +- ColumnarBroadcastExchange (29) + : +- ^ ProjectExecTransformer (27) + : +- ^ Scan parquet (26) + +- ^ InputIteratorTransformer (41) + +- ^ InputAdapter (40) + +- ^ BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ ProjectExecTransformer (36) + +- ^ Scan parquet (35) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (85) + +- HashAggregate (84) + +- Exchange (83) + +- HashAggregate (82) + +- Project (81) + +- BroadcastHashJoin Inner BuildRight (80) + :- Project (75) + : +- BroadcastHashJoin Inner BuildRight (74) + : :- Project (69) + : : +- BroadcastHashJoin Inner BuildLeft (68) + : : :- BroadcastExchange (56) + : : : +- Filter (55) + : : : +- Scan parquet (54) + : : +- BroadcastHashJoin LeftAnti BuildRight (67) + : : :- BroadcastHashJoin LeftSemi BuildRight (62) + : : : :- Project (59) + : : : : +- Filter (58) + : : : : +- Scan parquet (57) + : : : +- BroadcastExchange (61) + : : : +- Scan parquet (60) + : : +- BroadcastExchange (66) + : : +- Project (65) + : : +- Filter (64) + : : +- Scan parquet (63) + : +- BroadcastExchange (73) + : +- Project (72) + : +- Filter (71) + : +- Scan parquet (70) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- Scan parquet (76) (1) Scan parquet @@ -94,402 +89,382 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) Scan parquet +(7) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(10) ProjectExecTransformer +(8) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(11) Scan parquet +(9) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(11) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(12) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) BroadcastHashJoinExecTransformer +(15) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(18) Scan parquet +(16) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(19) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) - -(20) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(21) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(19) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(20) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(24) InputAdapter +(21) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(25) InputIteratorTransformer +(22) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(26) BroadcastHashJoinExecTransformer +(23) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(27) BroadcastHashJoinExecTransformer +(24) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(28) ProjectExecTransformer +(25) ProjectExecTransformer Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(29) Scan parquet +(26) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(30) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderstatus#X] -Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) - -(31) ProjectExecTransformer +(27) ProjectExecTransformer Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(32) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [1]: [o_orderkey#X] Arguments: false -(33) ColumnarBroadcastExchange +(29) ColumnarBroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(34) BroadcastQueryStage +(30) BroadcastQueryStage Output [1]: [o_orderkey#X] Arguments: X -(35) InputAdapter +(31) InputAdapter Input [1]: [o_orderkey#X] -(36) InputIteratorTransformer +(32) InputIteratorTransformer Input [1]: [o_orderkey#X] -(37) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(38) ProjectExecTransformer +(34) ProjectExecTransformer Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(39) Scan parquet +(35) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) - -(41) ProjectExecTransformer +(36) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(43) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(44) BroadcastQueryStage +(39) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(45) InputAdapter +(40) InputAdapter Input [1]: [n_nationkey#X] -(46) InputIteratorTransformer +(41) InputIteratorTransformer Input [1]: [n_nationkey#X] -(47) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(43) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(50) ProjectExecTransformer +(45) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(51) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(47) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(48) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(49) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(50) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(54) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(55) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(56) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(57) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(58) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(59) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(60) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(61) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(66) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(74) Project +(69) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(71) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(72) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(73) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(80) Project +(75) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(76) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(77) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(78) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(79) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) Project +(81) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(82) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(83) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(84) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(85) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(86) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt index 774b9ca85165..64084ebda520 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt @@ -1,40 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (25) + +- ^ SortExecTransformer (23) + +- ^ InputIteratorTransformer (22) + +- ^ InputAdapter (21) + +- ^ ShuffleQueryStage (20), Statistics(X) + +- ColumnarExchange (19) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FlushableHashAggregateExecTransformer (10) + +- ^ ProjectExecTransformer (9) + +- ^ BroadcastHashJoinExecTransformer LeftAnti (8) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (7) + +- ^ InputAdapter (6) + +- ^ BroadcastQueryStage (5), Statistics(X) + +- ColumnarBroadcastExchange (4) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (36) + +- Exchange (35) + +- HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin LeftAnti BuildRight (30) + :- Filter (27) + : +- Scan parquet (26) + +- BroadcastExchange (29) + +- Scan parquet (28) (1) Scan parquet @@ -44,300 +43,270 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) - -(3) Scan parquet +(2) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [1]: [o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(5) BroadcastQueryStage Output [1]: [o_custkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [o_custkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(8) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(10) ProjectExecTransformer +(9) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(11) FlushableHashAggregateExecTransformer +(10) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(19) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(20) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(21) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(22) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(23) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(26) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(27) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(28) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(29) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(31) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(32) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(33) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(34) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(35) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(36) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (48) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ^ InputAdapter (44) + +- ^ ShuffleQueryStage (43), Statistics(X) + +- ColumnarExchange (42) + +- ^ FlushableHashAggregateExecTransformer (40) + +- ^ ProjectExecTransformer (39) + +- ^ Scan parquet (38) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Project (51) + +- Filter (50) + +- Scan parquet (49) -(39) Scan parquet +(38) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) - -(41) ProjectExecTransformer +(39) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(40) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(42) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(43) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(44) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(45) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(48) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(49) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(50) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(51) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(52) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(53) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(54) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] -Arguments: isFinalPlan=true - -Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) -+- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) -+- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) \ No newline at end of file +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt index 6a5c8395fa5d..8b97940da5b5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt @@ -1,55 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (31) + +- TakeOrderedAndProjectExecTransformer (30) + +- ^ ProjectExecTransformer (28) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ BroadcastHashJoinExecTransformer Inner (18) + :- ^ ProjectExecTransformer (10) + : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ BroadcastQueryStage (15), Statistics(X) + +- ColumnarBroadcastExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ Scan parquet (11) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- Project (45) + +- BroadcastHashJoin Inner BuildRight (44) + :- Project (39) + : +- BroadcastHashJoin Inner BuildLeft (38) + : :- BroadcastExchange (35) + : : +- Project (34) + : : +- Filter (33) + : : +- Scan parquet (32) + : +- Filter (37) + : +- Scan parquet (36) + +- BroadcastExchange (43) + +- Project (42) + +- Filter (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,234 +56,222 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_mktsegment#X] -Arguments: ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [1]: [c_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(5) BroadcastQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(13) Scan parquet +(11) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) - -(15) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(17) ColumnarBroadcastExchange +(14) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(18) BroadcastQueryStage +(15) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(19) InputAdapter +(16) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(21) BroadcastHashJoinExecTransformer +(18) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(22) ProjectExecTransformer +(19) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(24) ProjectExecTransformer +(21) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, o_orderdate#X, o_shippriority#X, 42) AS hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(25) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(23) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(25) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(31) ProjectExecTransformer +(28) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(32) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(30) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(32) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(33) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(34) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(35) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(36) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(37) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(39) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(40) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(41) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(42) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(43) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(44) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(45) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(46) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(47) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(48) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(49) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt index 0bbf78a26ac8..31cba5c73b0a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt @@ -1,46 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- ^ InputAdapter (10) - +- ^ BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (10) + :- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (41) + +- Exchange (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- BroadcastHashJoin LeftSemi BuildRight (35) + :- Project (30) + : +- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -50,190 +48,182 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Arguments: ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(4) Scan parquet +(3) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(5) FilterExecTransformer -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) - -(6) ProjectExecTransformer +(4) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(7) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(8) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(9) BroadcastQueryStage +(7) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(10) InputAdapter +(8) InputAdapter Input [1]: [l_orderkey#X] -(11) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [l_orderkey#X] -(12) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(13) ProjectExecTransformer +(11) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(14) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(15) ProjectExecTransformer +(13) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(16) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(17) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(25) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(28) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(29) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(30) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(31) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(32) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(33) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(34) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(35) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(36) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(37) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(38) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(40) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(41) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt index cc6561aefb3b..dfdfede045a2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt @@ -1,98 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ^ InputAdapter (61) - +- ^ ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (59) + +- ^ SortExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ^ InputAdapter (55) + +- ^ ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ^ InputAdapter (49) + +- ^ ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner (42) + :- ^ ProjectExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : :- ^ ProjectExecTransformer (26) + : : +- ^ BroadcastHashJoinExecTransformer Inner (25) + : : :- ^ ProjectExecTransformer (18) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : : : : :- ^ InputIteratorTransformer (6) + : : : : : +- ^ InputAdapter (5) + : : : : : +- ^ BroadcastQueryStage (4), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (3) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (8) + : : : : +- ^ Scan parquet (7) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (24) + : : +- ^ InputAdapter (23) + : : +- ^ BroadcastQueryStage (22), Statistics(X) + : : +- ColumnarBroadcastExchange (21) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30), Statistics(X) + : +- ColumnarBroadcastExchange (29) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (41) + +- ^ InputAdapter (40) + +- ^ BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ ProjectExecTransformer (36) + +- ^ Scan parquet (35) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (77) + : : +- BroadcastHashJoin Inner BuildRight (76) + : : :- Project (72) + : : : +- BroadcastHashJoin Inner BuildRight (71) + : : : :- Project (67) + : : : : +- BroadcastHashJoin Inner BuildLeft (66) + : : : : :- BroadcastExchange (62) + : : : : : +- Filter (61) + : : : : : +- Scan parquet (60) + : : : : +- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- BroadcastExchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (80) + : +- Filter (79) + : +- Scan parquet (78) + +- BroadcastExchange (86) + +- Project (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -102,438 +96,414 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(8) Scan parquet +(7) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(10) ProjectExecTransformer +(8) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(13) Scan parquet +(11) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) - -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(21) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(19) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(24) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarBroadcastExchange +(21) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(26) BroadcastQueryStage +(22) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(23) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(24) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) BroadcastHashJoinExecTransformer +(25) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(30) ProjectExecTransformer +(26) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) Scan parquet +(27) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(32) FilterExecTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(33) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(34) ColumnarBroadcastExchange +(29) ColumnarBroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(30) BroadcastQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(36) InputAdapter +(31) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(37) InputIteratorTransformer +(32) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(38) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(39) ProjectExecTransformer +(34) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(40) Scan parquet +(35) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(41) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) - -(42) ProjectExecTransformer +(36) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(43) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(44) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(45) BroadcastQueryStage +(39) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(46) InputAdapter +(40) InputAdapter Input [1]: [r_regionkey#X] -(47) InputIteratorTransformer +(41) InputIteratorTransformer Input [1]: [r_regionkey#X] -(48) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(49) ProjectExecTransformer +(43) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(50) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(51) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(52) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(47) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(48) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(49) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(50) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(53) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(55) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(57) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(60) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(61) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(62) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(63) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(64) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(65) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(67) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(68) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(69) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(70) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(72) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(73) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(74) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(75) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(77) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(78) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(79) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(80) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(82) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(83) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(84) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(85) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(86) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(88) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(89) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(90) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(91) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(98) Exchange +(92) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(93) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt index 15f764040184..2e51f22a6b0e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt @@ -1,23 +1,22 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (18) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (11) + +- ^ RegularHashAggregateExecTransformer (9) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ ShuffleQueryStage (6), Statistics(X) + +- ColumnarExchange (5) + +- ^ FlushableHashAggregateExecTransformer (3) + +- ^ ProjectExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (17) + +- Exchange (16) + +- HashAggregate (15) + +- Project (14) + +- Filter (13) + +- Scan parquet (12) (1) Scan parquet @@ -27,86 +26,82 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(3) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(5) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(5) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(7) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(9) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(11) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(12) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(13) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(14) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(15) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(16) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(17) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(18) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt index bab148b6bb17..0a8c1111c851 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt @@ -1,92 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- ^ InputAdapter (17) - : : : +- ^ BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- ^ InputAdapter (26) - : : +- ^ BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ^ InputAdapter (51) + +- ^ ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ RegularHashAggregateExecTransformer (47) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FlushableHashAggregateExecTransformer (40) + +- ^ ProjectExecTransformer (39) + +- ^ BroadcastHashJoinExecTransformer Inner (38) + :- ^ ProjectExecTransformer (33) + : +- ^ BroadcastHashJoinExecTransformer Inner (32) + : :- ^ ProjectExecTransformer (25) + : : +- ^ BroadcastHashJoinExecTransformer Inner (24) + : : :- ^ ProjectExecTransformer (17) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (16) + : : : :- ^ ProjectExecTransformer (9) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (8) + : : : : :- ^ InputIteratorTransformer (6) + : : : : : +- ^ InputAdapter (5) + : : : : : +- ^ BroadcastQueryStage (4), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (3) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ Scan parquet (7) + : : : +- ^ InputIteratorTransformer (15) + : : : +- ^ InputAdapter (14) + : : : +- ^ BroadcastQueryStage (13), Statistics(X) + : : : +- ColumnarBroadcastExchange (12) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ BroadcastQueryStage (21), Statistics(X) + : : +- ColumnarBroadcastExchange (20) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ BroadcastQueryStage (29), Statistics(X) + : +- ColumnarBroadcastExchange (28) + : +- ^ Scan parquet (26) + +- ^ InputIteratorTransformer (37) + +- ^ InputAdapter (36) + +- ^ BroadcastQueryStage (35), Statistics(X) + +- ReusedExchange (34) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (87) + +- Exchange (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- BroadcastHashJoin Inner BuildRight (81) + :- Project (77) + : +- BroadcastHashJoin Inner BuildRight (76) + : :- Project (72) + : : +- BroadcastHashJoin Inner BuildRight (71) + : : :- Project (67) + : : : +- BroadcastHashJoin Inner BuildRight (66) + : : : :- Project (62) + : : : : +- BroadcastHashJoin Inner BuildLeft (61) + : : : : :- BroadcastExchange (58) + : : : : : +- Filter (57) + : : : : : +- Scan parquet (56) + : : : : +- Filter (60) + : : : : +- Scan parquet (59) + : : : +- BroadcastExchange (65) + : : : +- Filter (64) + : : : +- Scan parquet (63) + : : +- BroadcastExchange (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- BroadcastExchange (75) + : +- Filter (74) + : +- Scan parquet (73) + +- BroadcastExchange (80) + +- Filter (79) + +- Scan parquet (78) (1) Scan parquet @@ -96,406 +91,386 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) Scan parquet +(7) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(10) BroadcastHashJoinExecTransformer +(8) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) Scan parquet +(10) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(13) FilterExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) - -(14) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(15) ColumnarBroadcastExchange +(12) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(16) BroadcastQueryStage +(13) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(17) InputAdapter +(14) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(18) InputIteratorTransformer +(15) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(19) BroadcastHashJoinExecTransformer +(16) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(20) ProjectExecTransformer +(17) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(21) Scan parquet +(18) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(22) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(23) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(24) ColumnarBroadcastExchange +(20) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(25) BroadcastQueryStage +(21) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(26) InputAdapter +(22) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(27) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(28) BroadcastHashJoinExecTransformer +(24) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(29) ProjectExecTransformer +(25) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(30) Scan parquet +(26) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(31) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) - -(32) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(33) ColumnarBroadcastExchange +(28) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(34) BroadcastQueryStage +(29) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(35) InputAdapter +(30) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(36) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(37) BroadcastHashJoinExecTransformer +(32) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(38) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(39) ReusedExchange [Reuses operator id: 33] +(34) ReusedExchange [Reuses operator id: 28] Output [2]: [n_nationkey#X, n_name#X] -(40) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(41) InputAdapter +(36) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(42) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(43) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(44) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(45) FlushableHashAggregateExecTransformer +(40) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(45) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(47) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(49) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(50) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(51) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(52) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(53) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(56) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(57) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(58) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(59) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(60) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(61) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(62) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(63) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(64) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(65) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(67) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(72) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(73) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(74) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(75) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(77) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(80) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(82) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(83) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(84) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(85) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(86) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(87) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt index 757b1fcca6b4..6822e2becd51 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt @@ -1,125 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (121) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- ^ InputAdapter (18) - : : : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- ^ InputAdapter (27) - : : : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- ^ InputAdapter (36) - : : : +- ^ BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- ^ InputAdapter (54) - : +- ^ BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- ^ InputAdapter (64) - +- ^ BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (76) + +- ^ SortExecTransformer (74) + +- ^ InputIteratorTransformer (73) + +- ^ InputAdapter (72) + +- ^ ShuffleQueryStage (71), Statistics(X) + +- ColumnarExchange (70) + +- ^ ProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ InputIteratorTransformer (66) + +- ^ InputAdapter (65) + +- ^ ShuffleQueryStage (64), Statistics(X) + +- ColumnarExchange (63) + +- ^ ProjectExecTransformer (61) + +- ^ FlushableHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ BroadcastHashJoinExecTransformer Inner (58) + :- ^ ProjectExecTransformer (50) + : +- ^ BroadcastHashJoinExecTransformer Inner (49) + : :- ^ ProjectExecTransformer (42) + : : +- ^ BroadcastHashJoinExecTransformer Inner (41) + : : :- ^ ProjectExecTransformer (34) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : : : :- ^ ProjectExecTransformer (26) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (25) + : : : : :- ^ ProjectExecTransformer (18) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) + : : : : : :- ^ ProjectExecTransformer (10) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : : : : : : :- ^ InputIteratorTransformer (7) + : : : : : : : +- ^ InputAdapter (6) + : : : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (4) + : : : : : : : +- ^ ProjectExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ Scan parquet (8) + : : : : : +- ^ InputIteratorTransformer (16) + : : : : : +- ^ InputAdapter (15) + : : : : : +- ^ BroadcastQueryStage (14), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (13) + : : : : : +- ^ Scan parquet (11) + : : : : +- ^ InputIteratorTransformer (24) + : : : : +- ^ InputAdapter (23) + : : : : +- ^ BroadcastQueryStage (22), Statistics(X) + : : : : +- ColumnarBroadcastExchange (21) + : : : : +- ^ Scan parquet (19) + : : : +- ^ InputIteratorTransformer (32) + : : : +- ^ InputAdapter (31) + : : : +- ^ BroadcastQueryStage (30), Statistics(X) + : : : +- ColumnarBroadcastExchange (29) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (40) + : : +- ^ InputAdapter (39) + : : +- ^ BroadcastQueryStage (38), Statistics(X) + : : +- ColumnarBroadcastExchange (37) + : : +- ^ Scan parquet (35) + : +- ^ InputIteratorTransformer (48) + : +- ^ InputAdapter (47) + : +- ^ BroadcastQueryStage (46), Statistics(X) + : +- ColumnarBroadcastExchange (45) + : +- ^ Scan parquet (43) + +- ^ InputIteratorTransformer (57) + +- ^ InputAdapter (56) + +- ^ BroadcastQueryStage (55), Statistics(X) + +- ColumnarBroadcastExchange (54) + +- ^ ProjectExecTransformer (52) + +- ^ Scan parquet (51) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (120) + +- Exchange (119) + +- HashAggregate (118) + +- Exchange (117) + +- HashAggregate (116) + +- Project (115) + +- BroadcastHashJoin Inner BuildRight (114) + :- Project (109) + : +- BroadcastHashJoin Inner BuildRight (108) + : :- Project (104) + : : +- BroadcastHashJoin Inner BuildRight (103) + : : :- Project (99) + : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : :- Project (94) + : : : : +- BroadcastHashJoin Inner BuildRight (93) + : : : : :- Project (89) + : : : : : +- BroadcastHashJoin Inner BuildRight (88) + : : : : : :- Project (84) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (83) + : : : : : : :- BroadcastExchange (80) + : : : : : : : +- Project (79) + : : : : : : : +- Filter (78) + : : : : : : : +- Scan parquet (77) + : : : : : : +- Filter (82) + : : : : : : +- Scan parquet (81) + : : : : : +- BroadcastExchange (87) + : : : : : +- Filter (86) + : : : : : +- Scan parquet (85) + : : : : +- BroadcastExchange (92) + : : : : +- Filter (91) + : : : : +- Scan parquet (90) + : : : +- BroadcastExchange (97) + : : : +- Filter (96) + : : : +- Scan parquet (95) + : : +- BroadcastExchange (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- BroadcastExchange (107) + : +- Filter (106) + : +- Scan parquet (105) + +- BroadcastExchange (113) + +- Project (112) + +- Filter (111) + +- Scan parquet (110) (1) Scan parquet @@ -129,566 +121,534 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(5) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(13) Scan parquet +(11) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(21) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(22) Scan parquet +(19) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(23) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) - -(24) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(25) ColumnarBroadcastExchange +(21) ColumnarBroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(26) BroadcastQueryStage +(22) BroadcastQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(27) InputAdapter +(23) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(28) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(29) BroadcastHashJoinExecTransformer +(25) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(30) ProjectExecTransformer +(26) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(31) Scan parquet +(27) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(32) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(33) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(34) ColumnarBroadcastExchange +(29) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(30) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(36) InputAdapter +(31) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(37) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(38) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(39) ProjectExecTransformer +(34) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(40) Scan parquet +(35) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(41) FilterExecTransformer -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(42) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: false -(43) ColumnarBroadcastExchange +(37) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastQueryStage +(38) BroadcastQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(45) InputAdapter +(39) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(46) InputIteratorTransformer +(40) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(47) BroadcastHashJoinExecTransformer +(41) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(42) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(49) Scan parquet +(43) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(50) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(51) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(52) ColumnarBroadcastExchange +(45) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(53) BroadcastQueryStage +(46) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(54) InputAdapter +(47) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(55) InputIteratorTransformer +(48) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(56) BroadcastHashJoinExecTransformer +(49) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(57) ProjectExecTransformer +(50) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(58) Scan parquet +(51) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(59) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) - -(60) ProjectExecTransformer +(52) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(61) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(62) ColumnarBroadcastExchange +(54) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) BroadcastQueryStage +(55) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(64) InputAdapter +(56) InputAdapter Input [1]: [r_regionkey#X] -(65) InputIteratorTransformer +(57) InputIteratorTransformer Input [1]: [r_regionkey#X] -(66) BroadcastHashJoinExecTransformer +(58) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(67) ProjectExecTransformer +(59) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(68) FlushableHashAggregateExecTransformer +(60) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(69) ProjectExecTransformer +(61) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(70) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(63) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(64) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(65) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(66) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(68) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(69) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(70) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(71) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(72) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(73) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(74) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(76) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(77) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(78) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(79) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(80) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(81) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(82) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(84) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(85) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(86) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(87) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(89) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(90) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(91) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(92) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(94) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(95) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(96) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(97) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(99) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(100) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(101) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(102) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(105) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(106) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(107) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(110) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(111) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(112) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(113) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(114) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(115) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(116) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(117) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(118) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(127) Exchange +(119) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(120) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(121) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt index 4b0cb7140dcd..bec2b05cc2a3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt @@ -1,96 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ^ InputAdapter (60) - +- ^ ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (58) + +- ^ SortExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : :- ^ ProjectExecTransformer (26) + : : +- ^ BroadcastHashJoinExecTransformer Inner (25) + : : :- ^ ProjectExecTransformer (18) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ ProjectExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (24) + : : +- ^ InputAdapter (23) + : : +- ^ BroadcastQueryStage (22), Statistics(X) + : : +- ColumnarBroadcastExchange (21) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30), Statistics(X) + : +- ColumnarBroadcastExchange (29) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ Scan parquet (35) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (91) + +- Exchange (90) + +- HashAggregate (89) + +- Exchange (88) + +- HashAggregate (87) + +- Project (86) + +- BroadcastHashJoin Inner BuildRight (85) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (76) + : : +- BroadcastHashJoin Inner BuildRight (75) + : : :- Project (71) + : : : +- BroadcastHashJoin Inner BuildRight (70) + : : : :- Project (66) + : : : : +- BroadcastHashJoin Inner BuildLeft (65) + : : : : :- BroadcastExchange (62) + : : : : : +- Project (61) + : : : : : +- Filter (60) + : : : : : +- Scan parquet (59) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- BroadcastExchange (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- BroadcastExchange (79) + : +- Filter (78) + : +- Scan parquet (77) + +- BroadcastExchange (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -100,430 +94,406 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(5) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(8) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(13) Scan parquet +(11) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(21) ProjectExecTransformer +(18) ProjectExecTransformer Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(22) Scan parquet +(19) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(23) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) - -(24) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(25) ColumnarBroadcastExchange +(21) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(26) BroadcastQueryStage +(22) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(27) InputAdapter +(23) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(28) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(29) BroadcastHashJoinExecTransformer +(25) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(30) ProjectExecTransformer +(26) ProjectExecTransformer Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(31) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(32) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: isnotnull(o_orderkey#X) - -(33) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: false -(34) ColumnarBroadcastExchange +(29) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(30) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(36) InputAdapter +(31) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(37) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(38) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(39) ProjectExecTransformer +(34) ProjectExecTransformer Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(40) Scan parquet +(35) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(41) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(42) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarBroadcastExchange +(37) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastQueryStage +(38) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(39) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(40) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) BroadcastHashJoinExecTransformer +(41) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(42) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(44) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(46) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(47) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(48) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(49) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(52) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(53) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(54) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(55) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(56) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(59) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(60) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(61) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(62) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(63) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(64) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(65) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(66) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(67) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(68) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(69) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(70) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(71) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(72) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(73) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(74) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(76) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(77) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(78) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(79) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(81) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(84) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(85) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(86) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(87) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(88) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(89) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(90) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(91) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt index 634f26c86f24..0217849e5de2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt @@ -1,31 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (27) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (18) + +- ^ SortExecTransformer (16) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (3) + +- ^ ProjectExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (26) + +- Exchange (25) + +- HashAggregate (24) + +- Exchange (23) + +- HashAggregate (22) + +- Project (21) + +- Filter (20) + +- Scan parquet (19) (1) Scan parquet @@ -35,120 +34,116 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) FilterExecTransformer -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Arguments: (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X, ((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(3) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) ProjectExecTransformer +(4) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(6) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(6) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(7) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(8) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(9) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(12) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(13) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(14) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(15) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(16) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(18) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(19) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(20) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(21) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(22) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(23) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(24) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(25) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(26) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(27) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt index a0a2ead10a37..fc3503c42fef 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt @@ -1,68 +1,64 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ^ InputAdapter (37) - +- ^ ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- ^ InputAdapter (9) - : : +- ^ BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- ^ InputAdapter (19) - : +- ^ BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (39) + +- TakeOrderedAndProjectExecTransformer (38) + +- ^ ProjectExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ^ InputAdapter (33) + +- ^ ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- ^ ProjectExecTransformer (29) + +- ^ FlushableHashAggregateExecTransformer (28) + +- ^ ProjectExecTransformer (27) + +- ^ BroadcastHashJoinExecTransformer Inner (26) + :- ^ ProjectExecTransformer (19) + : +- ^ BroadcastHashJoinExecTransformer Inner (18) + : :- ^ ProjectExecTransformer (10) + : : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : : :- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (8) + : : +- ^ InputAdapter (7) + : : +- ^ BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ Scan parquet (2) + : +- ^ InputIteratorTransformer (17) + : +- ^ InputAdapter (16) + : +- ^ BroadcastQueryStage (15), Statistics(X) + : +- ColumnarBroadcastExchange (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ Scan parquet (11) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ BroadcastQueryStage (23), Statistics(X) + +- ColumnarBroadcastExchange (22) + +- ^ Scan parquet (20) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- BroadcastHashJoin Inner BuildRight (57) + :- Project (53) + : +- BroadcastHashJoin Inner BuildRight (52) + : :- Project (47) + : : +- BroadcastHashJoin Inner BuildRight (46) + : : :- Filter (41) + : : : +- Scan parquet (40) + : : +- BroadcastExchange (45) + : : +- Project (44) + : : +- Filter (43) + : : +- Scan parquet (42) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- BroadcastExchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,302 +68,286 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) Scan parquet +(2) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(4) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(5) ProjectExecTransformer +(3) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(6) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(7) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(8) BroadcastQueryStage +(6) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(9) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(10) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(13) Scan parquet +(11) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) - -(15) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(16) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(17) ColumnarBroadcastExchange +(14) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(18) BroadcastQueryStage +(15) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(19) InputAdapter +(16) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(21) BroadcastHashJoinExecTransformer +(18) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(22) ProjectExecTransformer +(19) ProjectExecTransformer Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) Scan parquet +(20) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(24) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(25) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(26) ColumnarBroadcastExchange +(22) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(27) BroadcastQueryStage +(23) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(28) InputAdapter +(24) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(29) InputIteratorTransformer +(25) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(30) BroadcastHashJoinExecTransformer +(26) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(31) ProjectExecTransformer +(27) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(28) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(33) ProjectExecTransformer +(29) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(34) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(31) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(32) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(33) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(34) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(40) ProjectExecTransformer +(36) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(41) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(38) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(40) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(41) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(45) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(46) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(51) Project +(47) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(48) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(49) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(50) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(51) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(57) Project +(53) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(55) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(56) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(62) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(59) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(60) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(61) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt index c81d951f6e31..943d59c56aaf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt @@ -1,59 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ^ InputAdapter (34) - +- ^ ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ^ InputAdapter (27) - +- ^ ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- ^ InputAdapter (8) - : +- ^ BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ FilterExecTransformer (27) + +- ^ RegularHashAggregateExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ FlushableHashAggregateExecTransformer (19) + +- ^ ProjectExecTransformer (18) + +- ^ BroadcastHashJoinExecTransformer Inner (17) + :- ^ ProjectExecTransformer (9) + : +- ^ BroadcastHashJoinExecTransformer Inner (8) + : :- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ BroadcastQueryStage (14), Statistics(X) + +- ColumnarBroadcastExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (54) + +- Exchange (53) + +- Filter (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- Project (42) + : +- BroadcastHashJoin Inner BuildRight (41) + : :- Filter (37) + : : +- Scan parquet (36) + : +- BroadcastExchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- BroadcastExchange (46) + +- Project (45) + +- Filter (44) + +- Scan parquet (43) (1) Scan parquet @@ -63,489 +60,472 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(3) Scan parquet +(2) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(4) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(5) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(7) BroadcastQueryStage +(5) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(8) InputAdapter +(6) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(9) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(10) BroadcastHashJoinExecTransformer +(8) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(12) Scan parquet +(10) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(13) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) - -(14) ProjectExecTransformer +(11) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [1]: [n_nationkey#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [1]: [n_nationkey#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(21) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(22) FlushableHashAggregateExecTransformer +(19) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(23) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(24) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(25) ColumnarExchange +(22) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(26) ShuffleQueryStage +(23) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(27) InputAdapter +(24) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(28) InputIteratorTransformer +(25) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(29) RegularHashAggregateExecTransformer +(26) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(30) FilterExecTransformer +(27) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(31) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(29) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(31) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(33) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(36) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(37) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(38) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(39) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(40) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(45) Project +(42) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(43) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(44) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(45) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(46) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(51) Project +(48) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(49) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(50) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(51) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(55) Filter +(52) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(53) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(54) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +Subquery:1 Hosting operator id = 27 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (95) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner (65) - : :- ^ FilterExecTransformer (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- ^ InputAdapter (63) - : +- ^ BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- ^ InputAdapter (69) - +- ^ BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ^ InputAdapter (73) + +- ^ ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ FlushableHashAggregateExecTransformer (69) + +- ^ ProjectExecTransformer (68) + +- ^ BroadcastHashJoinExecTransformer Inner (67) + :- ^ ProjectExecTransformer (62) + : +- ^ BroadcastHashJoinExecTransformer Inner (61) + : :- ^ Scan parquet (56) + : +- ^ InputIteratorTransformer (60) + : +- ^ InputAdapter (59) + : +- ^ BroadcastQueryStage (58), Statistics(X) + : +- ReusedExchange (57) + +- ^ InputIteratorTransformer (66) + +- ^ InputAdapter (65) + +- ^ BroadcastQueryStage (64), Statistics(X) + +- ReusedExchange (63) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (94) + +- Exchange (93) + +- HashAggregate (92) + +- Project (91) + +- BroadcastHashJoin Inner BuildRight (90) + :- Project (85) + : +- BroadcastHashJoin Inner BuildRight (84) + : :- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (83) + : +- Filter (82) + : +- Scan parquet (81) + +- BroadcastExchange (89) + +- Project (88) + +- Filter (87) + +- Scan parquet (86) + + +(56) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) FilterExecTransformer -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(61) ReusedExchange [Reuses operator id: 6] +(57) ReusedExchange [Reuses operator id: 4] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(58) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(59) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(60) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(61) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(66) ProjectExecTransformer +(62) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(63) ReusedExchange [Reuses operator id: 13] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(64) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(65) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(66) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(67) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(69) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(70) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(71) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(72) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(73) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(74) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(75) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(80) ProjectExecTransformer +(76) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(81) WholeStageCodegenTransformer (X) +(77) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(78) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(79) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(80) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(81) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(82) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(83) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(89) Project +(85) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(86) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(87) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(88) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(89) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(90) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(95) Project +(91) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(92) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(93) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(94) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(95) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt index 36ebd2bf5e76..f53f974558bd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt @@ -1,44 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ^ InputAdapter (18) - +- ^ ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (26) + +- ^ SortExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer Inner (9) + :- ^ InputIteratorTransformer (6) + : +- ^ InputAdapter (5) + : +- ^ BroadcastQueryStage (4), Statistics(X) + : +- ColumnarBroadcastExchange (3) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (8) + +- ^ Scan parquet (7) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (39) + +- Exchange (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- BroadcastHashJoin Inner BuildLeft (33) + :- BroadcastExchange (29) + : +- Filter (28) + : +- Scan parquet (27) + +- Project (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -48,184 +46,176 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: isnotnull(o_orderkey#X) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) Scan parquet +(7) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) - -(10) ProjectExecTransformer +(8) ProjectExecTransformer Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(13) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(14) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(15) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(16) ColumnarExchange +(14) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(17) ShuffleQueryStage +(15) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(18) InputAdapter +(16) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(19) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(20) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(24) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(26) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(28) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(29) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(30) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(31) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(32) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(33) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(36) Project +(34) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(35) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(36) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(37) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(38) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(39) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt index aa873582fab2..170698c552ac 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt @@ -1,53 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (51) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter (10) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftOuter (9) :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ ProjectExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (50) + +- Exchange (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- Exchange (44) + +- HashAggregate (43) + +- Project (42) + +- BroadcastHashJoin LeftOuter BuildRight (41) + :- Scan parquet (36) + +- BroadcastExchange (40) + +- Project (39) + +- Filter (38) + +- Scan parquet (37) (1) Scan parquet @@ -63,226 +62,222 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(3) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) - -(4) ProjectExecTransformer +(3) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(5) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(6) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(7) BroadcastQueryStage +(6) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(8) InputAdapter +(7) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(9) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(10) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(11) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(12) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(13) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, count#X] Input [2]: [c_custkey#X, count#X] -(14) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: false -(15) ColumnarExchange +(14) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, count#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [c_custkey#X, count#X] Arguments: X -(17) InputAdapter +(16) InputAdapter Input [2]: [c_custkey#X, count#X] -(18) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(19) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(21) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(25) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(29) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(31) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(33) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(36) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(37) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(38) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(39) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(40) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(43) Project +(42) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(43) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(44) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(46) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(47) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(48) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(49) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(50) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(51) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt index 3b5202f36c1e..b4a96c56cfb8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt @@ -1,37 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (32) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (20) + +- ^ ProjectExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer Inner (9) + :- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (31) + +- Exchange (30) + +- HashAggregate (29) + +- Project (28) + +- BroadcastHashJoin Inner BuildRight (27) + :- Project (23) + : +- Filter (22) + : +- Scan parquet (21) + +- BroadcastExchange (26) + +- Filter (25) + +- Scan parquet (24) (1) Scan parquet @@ -41,154 +39,146 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) Scan parquet +(3) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(5) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: isnotnull(p_partkey#X) - -(6) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [p_partkey#X, p_type#X] Arguments: false -(7) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(8) BroadcastQueryStage +(6) BroadcastQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(9) InputAdapter +(7) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(10) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(13) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(15) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(21) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(22) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(23) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(24) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(25) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(26) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(27) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(30) Project +(28) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(29) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(30) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(31) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(32) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt index 0fd34791c150..75af0c33004f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt @@ -1,44 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner (20) - :- ^ InputIteratorTransformer (7) - : +- ^ InputAdapter (6) - : +- ^ BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (24) + +- AQEShuffleRead (23) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ BroadcastHashJoinExecTransformer Inner (18) + :- ^ InputIteratorTransformer (6) + : +- ^ InputAdapter (5) + : +- ^ BroadcastQueryStage (4), Statistics(X) + : +- ColumnarBroadcastExchange (3) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (17) + +- ^ RegularHashAggregateExecTransformer (16) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ ProjectExecTransformer (10) + +- ^ FlushableHashAggregateExecTransformer (9) + +- ^ ProjectExecTransformer (8) + +- ^ Scan parquet (7) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (38) + +- Exchange (37) + +- Project (36) + +- BroadcastHashJoin Inner BuildLeft (35) + :- BroadcastExchange (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Filter (34) + +- HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- Filter (29) + +- Scan parquet (28) (1) Scan parquet @@ -48,339 +46,326 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: isnotnull(s_suppkey#X) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) Scan parquet +(7) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) - -(10) ProjectExecTransformer +(8) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) FlushableHashAggregateExecTransformer +(9) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(12) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(13) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(14) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(15) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(16) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(19) FilterExecTransformer +(17) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(20) BroadcastHashJoinExecTransformer +(18) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(23) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(25) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(26) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(27) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(28) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(29) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(30) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(31) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(32) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(33) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(36) Filter +(34) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(35) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(38) Project +(36) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(37) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(38) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +Subquery:1 Hosting operator id = 17 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ FilterExecTransformer (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ^ InputAdapter (47) + +- ^ ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ ProjectExecTransformer (43) + +- ^ FlushableHashAggregateExecTransformer (42) + +- ^ ProjectExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Exchange (59) + +- HashAggregate (58) + +- Project (57) + +- Filter (56) + +- Scan parquet (55) -(42) Scan parquet +(40) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) - -(44) ProjectExecTransformer +(41) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(42) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(43) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(45) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(46) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(47) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(48) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(53) ProjectExecTransformer +(50) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(54) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(55) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(56) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(57) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(58) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(59) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(60) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(64) HashAggregate +(61) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(62) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt index ed932e494f66..f87640ad9456 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt @@ -1,57 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (62) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ^ InputAdapter (31) - +- ^ ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (41) + +- ^ SortExecTransformer (39) + +- ^ InputIteratorTransformer (38) + +- ^ InputAdapter (37) + +- ^ ShuffleQueryStage (36), Statistics(X) + +- ColumnarExchange (35) + +- ^ RegularHashAggregateExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ ProjectExecTransformer (27) + +- ^ FlushableHashAggregateExecTransformer (26) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ ProjectExecTransformer (17) + +- ^ BroadcastHashJoinExecTransformer Inner (16) + :- ^ BroadcastHashJoinExecTransformer LeftAnti (9) + : :- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (8) + : +- ^ InputAdapter (7) + : +- ^ BroadcastQueryStage (6), Statistics(X) + : +- ColumnarBroadcastExchange (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ Scan parquet (2) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ BroadcastQueryStage (13), Statistics(X) + +- ColumnarBroadcastExchange (12) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (61) + +- Exchange (60) + +- HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- BroadcastHashJoin Inner BuildRight (52) + :- BroadcastHashJoin LeftAnti BuildRight (48) + : :- Filter (43) + : : +- Scan parquet (42) + : +- BroadcastExchange (47) + : +- Project (46) + : +- Filter (45) + : +- Scan parquet (44) + +- BroadcastExchange (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -61,255 +66,282 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: isnotnull(ps_partkey#X) +(2) Scan parquet +Output [2]: [s_suppkey#X, s_comment#X] +Batched: true +Location: InMemoryFileIndex [*] +PushedFilters: [IsNotNull(s_comment)] +ReadSchema: struct -(3) Scan parquet +(3) ProjectExecTransformer +Output [1]: [s_suppkey#X] +Input [2]: [s_suppkey#X, s_comment#X] + +(4) WholeStageCodegenTransformer (X) +Input [1]: [s_suppkey#X] +Arguments: false + +(5) ColumnarBroadcastExchange +Input [1]: [s_suppkey#X] +Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] + +(6) BroadcastQueryStage +Output [1]: [s_suppkey#X] +Arguments: X + +(7) InputAdapter +Input [1]: [s_suppkey#X] + +(8) InputIteratorTransformer +Input [1]: [s_suppkey#X] + +(9) BroadcastHashJoinExecTransformer +Left keys [1]: [ps_suppkey#X] +Right keys [1]: [s_suppkey#X] +Join type: LeftAnti +Join condition: None + +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(4) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) - -(5) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(6) ColumnarBroadcastExchange +(12) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(7) BroadcastQueryStage +(13) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(8) InputAdapter +(14) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(9) InputIteratorTransformer +(15) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(10) BroadcastHashJoinExecTransformer +(16) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(11) ProjectExecTransformer +(17) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(13) ProjectExecTransformer +(19) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(14) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(15) ColumnarExchange +(21) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(22) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(17) InputAdapter +(23) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(18) InputIteratorTransformer +(24) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(19) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) FlushableHashAggregateExecTransformer +(26) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(21) ProjectExecTransformer +(27) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(22) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(23) ColumnarExchange +(29) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(30) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(25) InputAdapter +(31) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(26) InputIteratorTransformer +(32) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(27) RegularHashAggregateExecTransformer +(33) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(28) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(35) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(36) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(37) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(38) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(39) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(41) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(42) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(43) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(45) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(46) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(47) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(43) Scan parquet +(49) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(50) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(51) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(47) Project +(53) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(54) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(57) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(58) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(59) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(61) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(62) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt index 0969c7267af5..9c4a028c442c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt @@ -1,86 +1,83 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (83) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ^ InputAdapter (48) - +- ^ ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : :- ^ InputIteratorTransformer (7) - : : +- ^ InputAdapter (6) - : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ^ InputAdapter (16) - : +- ^ ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- ^ InputAdapter (33) - +- ^ BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (50) + +- TakeOrderedAndProjectExecTransformer (49) + +- ^ RegularHashAggregateExecTransformer (47) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FlushableHashAggregateExecTransformer (40) + +- ^ ProjectExecTransformer (39) + +- ^ BroadcastHashJoinExecTransformer Inner (38) + :- ^ ProjectExecTransformer (26) + : +- ^ BroadcastHashJoinExecTransformer Inner (25) + : :- ^ InputIteratorTransformer (6) + : : +- ^ InputAdapter (5) + : : +- ^ BroadcastQueryStage (4), Statistics(X) + : : +- ColumnarBroadcastExchange (3) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (24) + : :- ^ Scan parquet (7) + : +- ^ InputIteratorTransformer (23) + : +- ^ InputAdapter (22) + : +- ^ BroadcastQueryStage (21), Statistics(X) + : +- ColumnarBroadcastExchange (20) + : +- ^ ProjectExecTransformer (18) + : +- ^ FilterExecTransformer (17) + : +- ^ RegularHashAggregateExecTransformer (16) + : +- ^ InputIteratorTransformer (15) + : +- ^ InputAdapter (14) + : +- ^ ShuffleQueryStage (13), Statistics(X) + : +- ColumnarExchange (12) + : +- ^ ProjectExecTransformer (10) + : +- ^ FlushableHashAggregateExecTransformer (9) + : +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (37) + +- ^ InputAdapter (36) + +- ^ BroadcastQueryStage (35), Statistics(X) + +- ColumnarBroadcastExchange (34) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (32) + :- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ BroadcastQueryStage (29), Statistics(X) + +- ReusedExchange (28) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (82) + +- HashAggregate (81) + +- Exchange (80) + +- HashAggregate (79) + +- Project (78) + +- BroadcastHashJoin Inner BuildRight (77) + :- Project (65) + : +- BroadcastHashJoin Inner BuildLeft (64) + : :- BroadcastExchange (53) + : : +- Filter (52) + : : +- Scan parquet (51) + : +- BroadcastHashJoin LeftSemi BuildRight (63) + : :- Filter (55) + : : +- Scan parquet (54) + : +- BroadcastExchange (62) + : +- Project (61) + : +- Filter (60) + : +- HashAggregate (59) + : +- Exchange (58) + : +- HashAggregate (57) + : +- Scan parquet (56) + +- BroadcastExchange (76) + +- BroadcastHashJoin LeftSemi BuildRight (75) + :- Filter (67) + : +- Scan parquet (66) + +- BroadcastExchange (74) + +- Project (73) + +- Filter (72) + +- HashAggregate (71) + +- Exchange (70) + +- HashAggregate (69) + +- Scan parquet (68) (1) Scan parquet @@ -90,395 +87,383 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_name#X] -Arguments: isnotnull(c_custkey#X) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_name#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(8) Scan parquet +(7) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) - -(10) Scan parquet +(8) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(9) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(13) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(12) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(13) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(14) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(15) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(16) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(17) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(20) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(21) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(24) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(27) BroadcastHashJoinExecTransformer +(25) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(28) ProjectExecTransformer +(26) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(27) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) FilterExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: isnotnull(l_orderkey#X) - -(31) ReusedExchange [Reuses operator id: 22] +(28) ReusedExchange [Reuses operator id: 20] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(29) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(30) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(31) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(32) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(36) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(34) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(36) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(42) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(40) FlushableHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(43) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(44) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(45) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(46) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(47) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(49) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(50) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(51) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(52) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(53) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(54) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(55) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(56) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(57) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(58) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(59) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(60) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(61) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(62) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(63) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(67) BroadcastHashJoin +(64) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(68) Project +(65) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(67) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(68) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(69) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(70) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(71) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(72) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(73) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(74) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(79) BroadcastExchange +(76) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(81) Project +(78) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(79) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(80) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(81) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(82) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(83) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt index 87f00043a52a..16e791cab81b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt @@ -1,36 +1,34 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (31) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (19) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer Inner (9) + :- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (30) + +- Exchange (29) + +- HashAggregate (28) + +- Project (27) + +- BroadcastHashJoin Inner BuildRight (26) + :- Project (22) + : +- Filter (21) + : +- Scan parquet (20) + +- BroadcastExchange (25) + +- Filter (24) + +- Scan parquet (23) (1) Scan parquet @@ -40,150 +38,142 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) FilterExecTransformer -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Arguments: ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(4) Scan parquet +(3) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(5) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) - -(6) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(7) ColumnarBroadcastExchange +(5) ColumnarBroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(8) BroadcastQueryStage +(6) BroadcastQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(9) InputAdapter +(7) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(10) InputIteratorTransformer +(8) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(11) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(13) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(14) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(15) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(19) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(20) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(21) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(22) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(23) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(24) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(25) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(26) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(27) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(28) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(29) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(30) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(31) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt index 5db32ebb4227..70067664658e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt @@ -1,96 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (96) +AdaptiveSparkPlan (91) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- AQEShuffleRead (59) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ BroadcastHashJoinExecTransformer Inner (54) - :- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (44) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (43) - : +- ^ InputAdapter (42) - : +- ^ BroadcastQueryStage (41), Statistics(X) - : +- ColumnarBroadcastExchange (40) - : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) - : :- ^ InputIteratorTransformer (18) - : : +- ^ InputAdapter (17) - : : +- ^ BroadcastQueryStage (16), Statistics(X) - : : +- ColumnarBroadcastExchange (15) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (13) - : : :- ^ FilterExecTransformer (4) - : : : +- ^ Scan parquet (3) - : : +- ^ InputIteratorTransformer (12) - : : +- ^ InputAdapter (11) - : : +- ^ BroadcastQueryStage (10), Statistics(X) - : : +- ColumnarBroadcastExchange (9) - : : +- ^ ProjectExecTransformer (7) - : : +- ^ FilterExecTransformer (6) - : : +- ^ Scan parquet (5) - : +- ^ FilterExecTransformer (36) - : +- ^ ProjectExecTransformer (35) - : +- ^ RegularHashAggregateExecTransformer (34) - : +- ^ InputIteratorTransformer (33) - : +- ^ InputAdapter (32) - : +- ^ ShuffleQueryStage (31), Statistics(X) - : +- ColumnarExchange (30) - : +- ^ ProjectExecTransformer (28) - : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi (26) - : :- ^ ProjectExecTransformer (21) - : : +- ^ FilterExecTransformer (20) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (25) - : +- ^ InputAdapter (24) - : +- ^ BroadcastQueryStage (23), Statistics(X) - : +- ReusedExchange (22) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ BroadcastQueryStage (51), Statistics(X) - +- ColumnarBroadcastExchange (50) - +- ^ ProjectExecTransformer (48) - +- ^ FilterExecTransformer (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (55) + +- AQEShuffleRead (54) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ BroadcastHashJoinExecTransformer Inner (49) + :- ^ ProjectExecTransformer (41) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (40) + : :- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (39) + : +- ^ InputAdapter (38) + : +- ^ BroadcastQueryStage (37), Statistics(X) + : +- ColumnarBroadcastExchange (36) + : +- ^ ProjectExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : :- ^ InputIteratorTransformer (15) + : : +- ^ InputAdapter (14) + : : +- ^ BroadcastQueryStage (13), Statistics(X) + : : +- ColumnarBroadcastExchange (12) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi (10) + : : :- ^ Scan parquet (2) + : : +- ^ InputIteratorTransformer (9) + : : +- ^ InputAdapter (8) + : : +- ^ BroadcastQueryStage (7), Statistics(X) + : : +- ColumnarBroadcastExchange (6) + : : +- ^ ProjectExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ FilterExecTransformer (32) + : +- ^ ProjectExecTransformer (31) + : +- ^ RegularHashAggregateExecTransformer (30) + : +- ^ InputIteratorTransformer (29) + : +- ^ InputAdapter (28) + : +- ^ ShuffleQueryStage (27), Statistics(X) + : +- ColumnarExchange (26) + : +- ^ ProjectExecTransformer (24) + : +- ^ FlushableHashAggregateExecTransformer (23) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi (22) + : :- ^ ProjectExecTransformer (17) + : : +- ^ Scan parquet (16) + : +- ^ InputIteratorTransformer (21) + : +- ^ InputAdapter (20) + : +- ^ BroadcastQueryStage (19), Statistics(X) + : +- ReusedExchange (18) + +- ^ InputIteratorTransformer (48) + +- ^ InputAdapter (47) + +- ^ BroadcastQueryStage (46), Statistics(X) + +- ColumnarBroadcastExchange (45) + +- ^ ProjectExecTransformer (43) + +- ^ Scan parquet (42) +- == Initial Plan == - Sort (95) - +- Exchange (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Filter (62) - : : +- Scan parquet (61) - : +- BroadcastExchange (85) - : +- Project (84) - : +- BroadcastHashJoin Inner BuildLeft (83) - : :- BroadcastExchange (70) - : : +- BroadcastHashJoin LeftSemi BuildRight (69) - : : :- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (68) - : : +- Project (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Filter (82) - : +- HashAggregate (81) - : +- Exchange (80) - : +- HashAggregate (79) - : +- BroadcastHashJoin LeftSemi BuildRight (78) - : :- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- BroadcastExchange (77) - : +- Project (76) - : +- Filter (75) - : +- Scan parquet (74) - +- BroadcastExchange (91) - +- Project (90) - +- Filter (89) - +- Scan parquet (88) + Sort (90) + +- Exchange (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (82) + : +- BroadcastHashJoin LeftSemi BuildRight (81) + : :- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastExchange (80) + : +- Project (79) + : +- BroadcastHashJoin Inner BuildLeft (78) + : :- BroadcastExchange (65) + : : +- BroadcastHashJoin LeftSemi BuildRight (64) + : : :- Filter (59) + : : : +- Scan parquet (58) + : : +- BroadcastExchange (63) + : : +- Project (62) + : : +- Filter (61) + : : +- Scan parquet (60) + : +- Filter (77) + : +- HashAggregate (76) + : +- Exchange (75) + : +- HashAggregate (74) + : +- BroadcastHashJoin LeftSemi BuildRight (73) + : :- Project (68) + : : +- Filter (67) + : : +- Scan parquet (66) + : +- BroadcastExchange (72) + : +- Project (71) + : +- Filter (70) + : +- Scan parquet (69) + +- BroadcastExchange (86) + +- Project (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -100,430 +95,410 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: isnotnull(s_nationkey#X) - -(3) Scan parquet +(2) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(4) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) - -(5) Scan parquet +(3) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(6) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) - -(7) ProjectExecTransformer +(4) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(8) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(9) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(10) BroadcastQueryStage +(7) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(11) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(12) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(13) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(14) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(15) ColumnarBroadcastExchange +(12) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(16) BroadcastQueryStage +(13) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(17) InputAdapter +(14) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(18) InputIteratorTransformer +(15) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(19) Scan parquet +(16) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(20) FilterExecTransformer -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) - -(21) ProjectExecTransformer +(17) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(22) ReusedExchange [Reuses operator id: 9] +(18) ReusedExchange [Reuses operator id: 6] Output [1]: [p_partkey#X] -(23) BroadcastQueryStage +(19) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(24) InputAdapter +(20) InputAdapter Input [1]: [p_partkey#X] -(25) InputIteratorTransformer +(21) InputIteratorTransformer Input [1]: [p_partkey#X] -(26) BroadcastHashJoinExecTransformer +(22) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(27) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(28) ProjectExecTransformer +(24) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(29) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(30) ColumnarExchange +(26) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(27) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(32) InputAdapter +(28) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(33) InputIteratorTransformer +(29) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(35) ProjectExecTransformer +(31) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(36) FilterExecTransformer +(32) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(37) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(38) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(39) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(40) ColumnarBroadcastExchange +(36) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(41) BroadcastQueryStage +(37) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(42) InputAdapter +(38) InputAdapter Input [1]: [ps_suppkey#X] -(43) InputIteratorTransformer +(39) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(44) BroadcastHashJoinExecTransformer +(40) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(45) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(46) Scan parquet +(42) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(47) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) - -(48) ProjectExecTransformer +(43) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(50) ColumnarBroadcastExchange +(45) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(51) BroadcastQueryStage +(46) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(52) InputAdapter +(47) InputAdapter Input [1]: [n_nationkey#X] -(53) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [n_nationkey#X] -(54) BroadcastHashJoinExecTransformer +(49) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(55) ProjectExecTransformer +(50) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(56) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(57) ColumnarExchange +(52) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(53) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(59) AQEShuffleRead +(54) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(60) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(61) Scan parquet +(56) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(57) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(63) Scan parquet +(58) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(64) Filter +(59) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(65) Scan parquet +(60) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(66) Filter +(61) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(67) Project +(62) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(63) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) BroadcastHashJoin +(64) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(70) BroadcastExchange +(65) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(71) Scan parquet +(66) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(72) Filter +(67) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(73) Project +(68) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(74) Scan parquet +(69) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(75) Filter +(70) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(76) Project +(71) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(77) BroadcastExchange +(72) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(79) HashAggregate +(74) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(80) Exchange +(75) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(76) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(82) Filter +(77) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(83) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(84) Project +(79) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(85) BroadcastExchange +(80) BroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(87) Project +(82) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(88) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(90) Project +(85) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(91) BroadcastExchange +(86) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(92) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) Project +(88) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(94) Exchange +(89) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Sort +(90) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(96) AdaptiveSparkPlan +(91) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt index 6e415d4a7c38..80e1d1dc5f36 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt @@ -1,90 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (86) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- ^ InputAdapter (6) - : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- ^ InputAdapter (24) - : : +- ^ BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ^ InputAdapter (49) + +- ^ ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner (42) + :- ^ ProjectExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : :- ^ ProjectExecTransformer (25) + : : +- ^ BroadcastHashJoinExecTransformer Inner (24) + : : :- ^ InputIteratorTransformer (6) + : : : +- ^ InputAdapter (5) + : : : +- ^ BroadcastQueryStage (4), Statistics(X) + : : : +- ColumnarBroadcastExchange (3) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti (23) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi (15) + : : : :- ^ ProjectExecTransformer (8) + : : : : +- ^ Scan parquet (7) + : : : +- ^ InputIteratorTransformer (14) + : : : +- ^ InputAdapter (13) + : : : +- ^ BroadcastQueryStage (12), Statistics(X) + : : : +- ColumnarBroadcastExchange (11) + : : : +- ^ Scan parquet (9) + : : +- ^ InputIteratorTransformer (22) + : : +- ^ InputAdapter (21) + : : +- ^ BroadcastQueryStage (20), Statistics(X) + : : +- ColumnarBroadcastExchange (19) + : : +- ^ ProjectExecTransformer (17) + : : +- ^ Scan parquet (16) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30), Statistics(X) + : +- ColumnarBroadcastExchange (29) + : +- ^ ProjectExecTransformer (27) + : +- ^ Scan parquet (26) + +- ^ InputIteratorTransformer (41) + +- ^ InputAdapter (40) + +- ^ BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ ProjectExecTransformer (36) + +- ^ Scan parquet (35) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (85) + +- HashAggregate (84) + +- Exchange (83) + +- HashAggregate (82) + +- Project (81) + +- BroadcastHashJoin Inner BuildRight (80) + :- Project (75) + : +- BroadcastHashJoin Inner BuildRight (74) + : :- Project (69) + : : +- BroadcastHashJoin Inner BuildLeft (68) + : : :- BroadcastExchange (56) + : : : +- Filter (55) + : : : +- Scan parquet (54) + : : +- BroadcastHashJoin LeftAnti BuildRight (67) + : : :- BroadcastHashJoin LeftSemi BuildRight (62) + : : : :- Project (59) + : : : : +- Filter (58) + : : : : +- Scan parquet (57) + : : : +- BroadcastExchange (61) + : : : +- Scan parquet (60) + : : +- BroadcastExchange (66) + : : +- Project (65) + : : +- Filter (64) + : : +- Scan parquet (63) + : +- BroadcastExchange (73) + : +- Project (72) + : +- Filter (71) + : +- Scan parquet (70) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- Scan parquet (76) (1) Scan parquet @@ -94,412 +89,392 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) Scan parquet +(7) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(10) ProjectExecTransformer +(8) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(11) Scan parquet +(9) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarBroadcastExchange +(11) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(14) BroadcastQueryStage +(12) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) BroadcastHashJoinExecTransformer +(15) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(18) Scan parquet +(16) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(19) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) - -(20) ProjectExecTransformer +(17) ProjectExecTransformer Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(21) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(19) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(20) BroadcastQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(24) InputAdapter +(21) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(25) InputIteratorTransformer +(22) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(26) BroadcastHashJoinExecTransformer +(23) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(27) BroadcastHashJoinExecTransformer +(24) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(28) ProjectExecTransformer +(25) ProjectExecTransformer Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(29) Scan parquet +(26) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(30) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderstatus#X] -Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) - -(31) ProjectExecTransformer +(27) ProjectExecTransformer Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(32) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [1]: [o_orderkey#X] Arguments: false -(33) ColumnarBroadcastExchange +(29) ColumnarBroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(34) BroadcastQueryStage +(30) BroadcastQueryStage Output [1]: [o_orderkey#X] Arguments: X -(35) InputAdapter +(31) InputAdapter Input [1]: [o_orderkey#X] -(36) InputIteratorTransformer +(32) InputIteratorTransformer Input [1]: [o_orderkey#X] -(37) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(38) ProjectExecTransformer +(34) ProjectExecTransformer Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(39) Scan parquet +(35) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) - -(41) ProjectExecTransformer +(36) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(43) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(44) BroadcastQueryStage +(39) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(45) InputAdapter +(40) InputAdapter Input [1]: [n_nationkey#X] -(46) InputIteratorTransformer +(41) InputIteratorTransformer Input [1]: [n_nationkey#X] -(47) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(43) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(50) ProjectExecTransformer +(45) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(51) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(47) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(48) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(49) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(50) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(53) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(54) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(55) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(56) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(57) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(58) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(59) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(60) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(61) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(66) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(74) Project +(69) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(70) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(71) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(72) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(73) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(80) Project +(75) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(76) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(77) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(78) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(79) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(86) Project +(81) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(82) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(83) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(84) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(85) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(86) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt index b0b5964d0a94..503fe27ab555 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt @@ -1,40 +1,39 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ^ InputAdapter (22) - +- ^ ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ^ InputAdapter (16) - +- ^ ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- ^ InputAdapter (7) - +- ^ BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (25) + +- ^ SortExecTransformer (23) + +- ^ InputIteratorTransformer (22) + +- ^ InputAdapter (21) + +- ^ ShuffleQueryStage (20), Statistics(X) + +- ColumnarExchange (19) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FlushableHashAggregateExecTransformer (10) + +- ^ ProjectExecTransformer (9) + +- ^ BroadcastHashJoinExecTransformer LeftAnti (8) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (7) + +- ^ InputAdapter (6) + +- ^ BroadcastQueryStage (5), Statistics(X) + +- ColumnarBroadcastExchange (4) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (36) + +- Exchange (35) + +- HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin LeftAnti BuildRight (30) + :- Filter (27) + : +- Scan parquet (26) + +- BroadcastExchange (29) + +- Scan parquet (28) (1) Scan parquet @@ -44,302 +43,272 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) - -(3) Scan parquet +(2) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [1]: [o_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(5) BroadcastQueryStage Output [1]: [o_custkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [o_custkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [o_custkey#X] -(9) BroadcastHashJoinExecTransformer +(8) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(10) ProjectExecTransformer +(9) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(11) FlushableHashAggregateExecTransformer +(10) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(12) ProjectExecTransformer +(11) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(13) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(13) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(14) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(15) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(19) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(19) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(20) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(21) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(22) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(23) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(26) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(27) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(28) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(29) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(32) Project +(31) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(32) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(33) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(34) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(35) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(36) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (48) + +- ^ RegularHashAggregateExecTransformer (46) + +- ^ InputIteratorTransformer (45) + +- ^ InputAdapter (44) + +- ^ ShuffleQueryStage (43), Statistics(X) + +- ColumnarExchange (42) + +- ^ FlushableHashAggregateExecTransformer (40) + +- ^ ProjectExecTransformer (39) + +- ^ Scan parquet (38) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- Project (51) + +- Filter (50) + +- Scan parquet (49) -(39) Scan parquet +(38) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) - -(41) ProjectExecTransformer +(39) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(40) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(42) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(43) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(44) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(45) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(46) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(48) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(49) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(50) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(51) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(52) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(53) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(54) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] -Arguments: isFinalPlan=true - -Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) -+- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) -+- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) \ No newline at end of file +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt index 6ce73eb0fa38..cd21047a80a5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt @@ -1,55 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (50) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ^ InputAdapter (28) - +- ^ ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (31) + +- TakeOrderedAndProjectExecTransformer (30) + +- ^ ProjectExecTransformer (28) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ BroadcastHashJoinExecTransformer Inner (18) + :- ^ ProjectExecTransformer (10) + : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (17) + +- ^ InputAdapter (16) + +- ^ BroadcastQueryStage (15), Statistics(X) + +- ColumnarBroadcastExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ Scan parquet (11) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- Project (45) + +- BroadcastHashJoin Inner BuildRight (44) + :- Project (39) + : +- BroadcastHashJoin Inner BuildLeft (38) + : :- BroadcastExchange (35) + : : +- Project (34) + : : +- Filter (33) + : : +- Scan parquet (32) + : +- Filter (37) + : +- Scan parquet (36) + +- BroadcastExchange (43) + +- Project (42) + +- Filter (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,238 +56,226 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_mktsegment#X] -Arguments: ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [1]: [c_custkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(5) BroadcastQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(13) Scan parquet +(11) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) - -(15) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(17) ColumnarBroadcastExchange +(14) ColumnarBroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(18) BroadcastQueryStage +(15) BroadcastQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(19) InputAdapter +(16) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(20) InputIteratorTransformer +(17) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(21) BroadcastHashJoinExecTransformer +(18) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(22) ProjectExecTransformer +(19) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(24) ProjectExecTransformer +(21) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, o_orderdate#X, o_shippriority#X, 42) AS hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(25) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(23) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(25) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(31) ProjectExecTransformer +(28) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(32) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(30) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(31) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(32) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(33) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(34) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(35) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(36) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(37) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(42) Project +(39) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(40) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(41) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(42) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(43) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(44) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(48) Project +(45) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(46) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(47) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(48) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(49) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(50) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt index 2401dc071221..95367083053d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt @@ -1,46 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ^ InputAdapter (25) - +- ^ ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ^ InputAdapter (19) - +- ^ ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- ^ InputAdapter (10) - +- ^ BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ^ InputAdapter (17) + +- ^ ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftSemi (10) + :- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (41) + +- Exchange (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- BroadcastHashJoin LeftSemi BuildRight (35) + :- Project (30) + : +- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -50,192 +48,184 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Arguments: ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(4) Scan parquet +(3) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(5) FilterExecTransformer -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) - -(6) ProjectExecTransformer +(4) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(7) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(8) ColumnarBroadcastExchange +(6) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(9) BroadcastQueryStage +(7) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(10) InputAdapter +(8) InputAdapter Input [1]: [l_orderkey#X] -(11) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [l_orderkey#X] -(12) BroadcastHashJoinExecTransformer +(10) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(13) ProjectExecTransformer +(11) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(14) FlushableHashAggregateExecTransformer +(12) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(15) ProjectExecTransformer +(13) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(16) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(17) ColumnarExchange +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(18) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(19) InputAdapter +(17) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(20) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(21) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(21) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(22) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(23) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(24) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(25) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(28) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(29) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(30) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(31) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(32) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(33) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(34) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(35) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(38) Project +(36) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(37) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(38) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(39) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(40) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(41) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt index e811b6370366..71c8049fa4dd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt @@ -1,98 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ^ InputAdapter (61) - +- ^ ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ^ InputAdapter (55) - +- ^ ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (59) + +- ^ SortExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ^ InputAdapter (55) + +- ^ ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ^ InputAdapter (49) + +- ^ ShuffleQueryStage (48), Statistics(X) + +- ColumnarExchange (47) + +- ^ ProjectExecTransformer (45) + +- ^ FlushableHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner (42) + :- ^ ProjectExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : :- ^ ProjectExecTransformer (26) + : : +- ^ BroadcastHashJoinExecTransformer Inner (25) + : : :- ^ ProjectExecTransformer (18) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : : : : :- ^ InputIteratorTransformer (6) + : : : : : +- ^ InputAdapter (5) + : : : : : +- ^ BroadcastQueryStage (4), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (3) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (8) + : : : : +- ^ Scan parquet (7) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (24) + : : +- ^ InputAdapter (23) + : : +- ^ BroadcastQueryStage (22), Statistics(X) + : : +- ColumnarBroadcastExchange (21) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30), Statistics(X) + : +- ColumnarBroadcastExchange (29) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (41) + +- ^ InputAdapter (40) + +- ^ BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ ProjectExecTransformer (36) + +- ^ Scan parquet (35) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (77) + : : +- BroadcastHashJoin Inner BuildRight (76) + : : :- Project (72) + : : : +- BroadcastHashJoin Inner BuildRight (71) + : : : :- Project (67) + : : : : +- BroadcastHashJoin Inner BuildLeft (66) + : : : : :- BroadcastExchange (62) + : : : : : +- Filter (61) + : : : : : +- Scan parquet (60) + : : : : +- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (70) + : : : +- Filter (69) + : : : +- Scan parquet (68) + : : +- BroadcastExchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- BroadcastExchange (80) + : +- Filter (79) + : +- Scan parquet (78) + +- BroadcastExchange (86) + +- Project (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -102,448 +96,424 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(8) Scan parquet +(7) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(10) ProjectExecTransformer +(8) ProjectExecTransformer Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(13) Scan parquet +(11) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) - -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(21) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(22) Scan parquet +(19) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(23) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(24) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(25) ColumnarBroadcastExchange +(21) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(26) BroadcastQueryStage +(22) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(27) InputAdapter +(23) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(28) InputIteratorTransformer +(24) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(29) BroadcastHashJoinExecTransformer +(25) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(26) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(31) Scan parquet +(27) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(32) FilterExecTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(33) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(34) ColumnarBroadcastExchange +(29) ColumnarBroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(30) BroadcastQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(36) InputAdapter +(31) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(37) InputIteratorTransformer +(32) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(38) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(39) ProjectExecTransformer +(34) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(40) Scan parquet +(35) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(41) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) - -(42) ProjectExecTransformer +(36) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(43) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(44) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(45) BroadcastQueryStage +(39) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(46) InputAdapter +(40) InputAdapter Input [1]: [r_regionkey#X] -(47) InputIteratorTransformer +(41) InputIteratorTransformer Input [1]: [r_regionkey#X] -(48) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(49) ProjectExecTransformer +(43) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(50) FlushableHashAggregateExecTransformer +(44) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(51) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(52) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(53) ColumnarExchange +(47) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(54) ShuffleQueryStage +(48) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(55) InputAdapter +(49) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(56) InputIteratorTransformer +(50) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(57) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(58) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(53) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(55) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(57) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(60) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(61) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(62) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(63) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(64) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(65) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(73) Project +(67) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(68) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(69) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(70) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(78) Project +(72) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(73) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(74) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(75) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(83) Project +(77) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(78) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(79) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(80) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) Project +(82) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(83) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(84) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(85) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(86) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(94) Project +(88) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(89) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(90) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(91) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(98) Exchange +(92) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(93) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt index 0882ff9e151c..733adf0d0b4a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt @@ -1,23 +1,22 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (18) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (11) + +- ^ RegularHashAggregateExecTransformer (9) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ ShuffleQueryStage (6), Statistics(X) + +- ColumnarExchange (5) + +- ^ FlushableHashAggregateExecTransformer (3) + +- ^ ProjectExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (17) + +- Exchange (16) + +- HashAggregate (15) + +- Project (14) + +- Filter (13) + +- Scan parquet (12) (1) Scan parquet @@ -27,86 +26,82 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(3) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(5) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(5) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(7) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(9) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(11) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(12) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(13) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(14) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(15) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(16) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(17) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(18) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt index 129da90d3a2b..93a86cb78a95 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt @@ -1,92 +1,87 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ^ InputAdapter (56) - +- ^ ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ^ InputAdapter (50) - +- ^ ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- ^ InputAdapter (6) - : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- ^ InputAdapter (17) - : : : +- ^ BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- ^ InputAdapter (26) - : : +- ^ BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- ^ InputAdapter (35) - : +- ^ BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- ^ InputAdapter (41) - +- ^ BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ^ InputAdapter (51) + +- ^ ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ RegularHashAggregateExecTransformer (47) + +- ^ InputIteratorTransformer (46) + +- ^ InputAdapter (45) + +- ^ ShuffleQueryStage (44), Statistics(X) + +- ColumnarExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FlushableHashAggregateExecTransformer (40) + +- ^ ProjectExecTransformer (39) + +- ^ BroadcastHashJoinExecTransformer Inner (38) + :- ^ ProjectExecTransformer (33) + : +- ^ BroadcastHashJoinExecTransformer Inner (32) + : :- ^ ProjectExecTransformer (25) + : : +- ^ BroadcastHashJoinExecTransformer Inner (24) + : : :- ^ ProjectExecTransformer (17) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (16) + : : : :- ^ ProjectExecTransformer (9) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (8) + : : : : :- ^ InputIteratorTransformer (6) + : : : : : +- ^ InputAdapter (5) + : : : : : +- ^ BroadcastQueryStage (4), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (3) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ Scan parquet (7) + : : : +- ^ InputIteratorTransformer (15) + : : : +- ^ InputAdapter (14) + : : : +- ^ BroadcastQueryStage (13), Statistics(X) + : : : +- ColumnarBroadcastExchange (12) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (23) + : : +- ^ InputAdapter (22) + : : +- ^ BroadcastQueryStage (21), Statistics(X) + : : +- ColumnarBroadcastExchange (20) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (31) + : +- ^ InputAdapter (30) + : +- ^ BroadcastQueryStage (29), Statistics(X) + : +- ColumnarBroadcastExchange (28) + : +- ^ Scan parquet (26) + +- ^ InputIteratorTransformer (37) + +- ^ InputAdapter (36) + +- ^ BroadcastQueryStage (35), Statistics(X) + +- ReusedExchange (34) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (87) + +- Exchange (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Project (82) + +- BroadcastHashJoin Inner BuildRight (81) + :- Project (77) + : +- BroadcastHashJoin Inner BuildRight (76) + : :- Project (72) + : : +- BroadcastHashJoin Inner BuildRight (71) + : : :- Project (67) + : : : +- BroadcastHashJoin Inner BuildRight (66) + : : : :- Project (62) + : : : : +- BroadcastHashJoin Inner BuildLeft (61) + : : : : :- BroadcastExchange (58) + : : : : : +- Filter (57) + : : : : : +- Scan parquet (56) + : : : : +- Filter (60) + : : : : +- Scan parquet (59) + : : : +- BroadcastExchange (65) + : : : +- Filter (64) + : : : +- Scan parquet (63) + : : +- BroadcastExchange (70) + : : +- Filter (69) + : : +- Scan parquet (68) + : +- BroadcastExchange (75) + : +- Filter (74) + : +- Scan parquet (73) + +- BroadcastExchange (80) + +- Filter (79) + +- Scan parquet (78) (1) Scan parquet @@ -96,416 +91,396 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) WholeStageCodegenTransformer (X) +(2) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(4) ColumnarBroadcastExchange +(3) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(5) BroadcastQueryStage +(4) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(6) InputAdapter +(5) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(7) InputIteratorTransformer +(6) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(8) Scan parquet +(7) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(10) BroadcastHashJoinExecTransformer +(8) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) Scan parquet +(10) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(13) FilterExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) - -(14) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_custkey#X] Arguments: false -(15) ColumnarBroadcastExchange +(12) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(16) BroadcastQueryStage +(13) BroadcastQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(17) InputAdapter +(14) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(18) InputIteratorTransformer +(15) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(19) BroadcastHashJoinExecTransformer +(16) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(20) ProjectExecTransformer +(17) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(21) Scan parquet +(18) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(22) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(23) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(24) ColumnarBroadcastExchange +(20) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(25) BroadcastQueryStage +(21) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(26) InputAdapter +(22) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(27) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(28) BroadcastHashJoinExecTransformer +(24) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(29) ProjectExecTransformer +(25) ProjectExecTransformer Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(30) Scan parquet +(26) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(31) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) - -(32) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(33) ColumnarBroadcastExchange +(28) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(34) BroadcastQueryStage +(29) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(35) InputAdapter +(30) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(36) InputIteratorTransformer +(31) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(37) BroadcastHashJoinExecTransformer +(32) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(38) ProjectExecTransformer +(33) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(39) ReusedExchange [Reuses operator id: 33] +(34) ReusedExchange [Reuses operator id: 28] Output [2]: [n_nationkey#X, n_name#X] -(40) BroadcastQueryStage +(35) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(41) InputAdapter +(36) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(42) InputIteratorTransformer +(37) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(43) BroadcastHashJoinExecTransformer +(38) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(44) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(45) FlushableHashAggregateExecTransformer +(40) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(41) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(43) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(44) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(45) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(46) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(47) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(53) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(49) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(50) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(51) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(52) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(53) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(56) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(57) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(58) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(59) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(60) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(61) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(67) Project +(62) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(63) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(64) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(65) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(66) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(72) Project +(67) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(68) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(69) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(70) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(71) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(77) Project +(72) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(73) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(74) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(75) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(76) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(77) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(79) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(80) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(82) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(83) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(84) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(85) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(86) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(87) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt index 4ac782958199..caddcc769034 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt @@ -1,125 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (121) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ^ InputAdapter (73) - +- ^ ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- ^ InputAdapter (18) - : : : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- ^ InputAdapter (27) - : : : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- ^ InputAdapter (36) - : : : +- ^ BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- ^ InputAdapter (54) - : +- ^ BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- ^ InputAdapter (64) - +- ^ BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (76) + +- ^ SortExecTransformer (74) + +- ^ InputIteratorTransformer (73) + +- ^ InputAdapter (72) + +- ^ ShuffleQueryStage (71), Statistics(X) + +- ColumnarExchange (70) + +- ^ ProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (67) + +- ^ InputIteratorTransformer (66) + +- ^ InputAdapter (65) + +- ^ ShuffleQueryStage (64), Statistics(X) + +- ColumnarExchange (63) + +- ^ ProjectExecTransformer (61) + +- ^ FlushableHashAggregateExecTransformer (60) + +- ^ ProjectExecTransformer (59) + +- ^ BroadcastHashJoinExecTransformer Inner (58) + :- ^ ProjectExecTransformer (50) + : +- ^ BroadcastHashJoinExecTransformer Inner (49) + : :- ^ ProjectExecTransformer (42) + : : +- ^ BroadcastHashJoinExecTransformer Inner (41) + : : :- ^ ProjectExecTransformer (34) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : : : :- ^ ProjectExecTransformer (26) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (25) + : : : : :- ^ ProjectExecTransformer (18) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) + : : : : : :- ^ ProjectExecTransformer (10) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : : : : : : :- ^ InputIteratorTransformer (7) + : : : : : : : +- ^ InputAdapter (6) + : : : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (4) + : : : : : : : +- ^ ProjectExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ Scan parquet (8) + : : : : : +- ^ InputIteratorTransformer (16) + : : : : : +- ^ InputAdapter (15) + : : : : : +- ^ BroadcastQueryStage (14), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (13) + : : : : : +- ^ Scan parquet (11) + : : : : +- ^ InputIteratorTransformer (24) + : : : : +- ^ InputAdapter (23) + : : : : +- ^ BroadcastQueryStage (22), Statistics(X) + : : : : +- ColumnarBroadcastExchange (21) + : : : : +- ^ Scan parquet (19) + : : : +- ^ InputIteratorTransformer (32) + : : : +- ^ InputAdapter (31) + : : : +- ^ BroadcastQueryStage (30), Statistics(X) + : : : +- ColumnarBroadcastExchange (29) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (40) + : : +- ^ InputAdapter (39) + : : +- ^ BroadcastQueryStage (38), Statistics(X) + : : +- ColumnarBroadcastExchange (37) + : : +- ^ Scan parquet (35) + : +- ^ InputIteratorTransformer (48) + : +- ^ InputAdapter (47) + : +- ^ BroadcastQueryStage (46), Statistics(X) + : +- ColumnarBroadcastExchange (45) + : +- ^ Scan parquet (43) + +- ^ InputIteratorTransformer (57) + +- ^ InputAdapter (56) + +- ^ BroadcastQueryStage (55), Statistics(X) + +- ColumnarBroadcastExchange (54) + +- ^ ProjectExecTransformer (52) + +- ^ Scan parquet (51) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (120) + +- Exchange (119) + +- HashAggregate (118) + +- Exchange (117) + +- HashAggregate (116) + +- Project (115) + +- BroadcastHashJoin Inner BuildRight (114) + :- Project (109) + : +- BroadcastHashJoin Inner BuildRight (108) + : :- Project (104) + : : +- BroadcastHashJoin Inner BuildRight (103) + : : :- Project (99) + : : : +- BroadcastHashJoin Inner BuildRight (98) + : : : :- Project (94) + : : : : +- BroadcastHashJoin Inner BuildRight (93) + : : : : :- Project (89) + : : : : : +- BroadcastHashJoin Inner BuildRight (88) + : : : : : :- Project (84) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (83) + : : : : : : :- BroadcastExchange (80) + : : : : : : : +- Project (79) + : : : : : : : +- Filter (78) + : : : : : : : +- Scan parquet (77) + : : : : : : +- Filter (82) + : : : : : : +- Scan parquet (81) + : : : : : +- BroadcastExchange (87) + : : : : : +- Filter (86) + : : : : : +- Scan parquet (85) + : : : : +- BroadcastExchange (92) + : : : : +- Filter (91) + : : : : +- Scan parquet (90) + : : : +- BroadcastExchange (97) + : : : +- Filter (96) + : : : +- Scan parquet (95) + : : +- BroadcastExchange (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- BroadcastExchange (107) + : +- Filter (106) + : +- Scan parquet (105) + +- BroadcastExchange (113) + +- Project (112) + +- Filter (111) + +- Scan parquet (110) (1) Scan parquet @@ -129,580 +121,548 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(5) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(13) Scan parquet +(11) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(21) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(22) Scan parquet +(19) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(23) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) - -(24) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(25) ColumnarBroadcastExchange +(21) ColumnarBroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(26) BroadcastQueryStage +(22) BroadcastQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(27) InputAdapter +(23) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(28) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(29) BroadcastHashJoinExecTransformer +(25) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(26) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(31) Scan parquet +(27) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(32) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(33) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [c_custkey#X, c_nationkey#X] Arguments: false -(34) ColumnarBroadcastExchange +(29) ColumnarBroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(30) BroadcastQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(36) InputAdapter +(31) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(37) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(38) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(39) ProjectExecTransformer +(34) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(40) Scan parquet +(35) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(41) FilterExecTransformer -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(42) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: false -(43) ColumnarBroadcastExchange +(37) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastQueryStage +(38) BroadcastQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(45) InputAdapter +(39) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(46) InputIteratorTransformer +(40) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(47) BroadcastHashJoinExecTransformer +(41) BroadcastHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(42) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(49) Scan parquet +(43) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(50) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(51) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(52) ColumnarBroadcastExchange +(45) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(53) BroadcastQueryStage +(46) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(54) InputAdapter +(47) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(55) InputIteratorTransformer +(48) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(56) BroadcastHashJoinExecTransformer +(49) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(57) ProjectExecTransformer +(50) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(58) Scan parquet +(51) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(59) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) - -(60) ProjectExecTransformer +(52) ProjectExecTransformer Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(61) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [1]: [r_regionkey#X] Arguments: false -(62) ColumnarBroadcastExchange +(54) ColumnarBroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(63) BroadcastQueryStage +(55) BroadcastQueryStage Output [1]: [r_regionkey#X] Arguments: X -(64) InputAdapter +(56) InputAdapter Input [1]: [r_regionkey#X] -(65) InputIteratorTransformer +(57) InputIteratorTransformer Input [1]: [r_regionkey#X] -(66) BroadcastHashJoinExecTransformer +(58) BroadcastHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(67) ProjectExecTransformer +(59) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(68) FlushableHashAggregateExecTransformer +(60) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(69) ProjectExecTransformer +(61) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(70) WholeStageCodegenTransformer (X) +(62) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(71) ColumnarExchange +(63) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(72) ShuffleQueryStage +(64) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(73) InputAdapter +(65) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(74) InputIteratorTransformer +(66) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(75) RegularHashAggregateExecTransformer +(67) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(76) ProjectExecTransformer +(68) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(77) WholeStageCodegenTransformer (X) +(69) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(70) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(71) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(72) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(73) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(74) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(76) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(77) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(78) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(79) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(80) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(81) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(82) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(92) Project +(84) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(85) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(86) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(87) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(97) Project +(89) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(90) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(91) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(92) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(102) Project +(94) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(95) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(96) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(97) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(98) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(107) Project +(99) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(100) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(101) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(102) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(103) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(112) Project +(104) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(105) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(106) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(107) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(108) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(109) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(110) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(111) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(112) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(113) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(114) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(123) Project +(115) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(116) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(117) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(118) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(127) Exchange +(119) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(120) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(121) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt index 1a41ac753c5c..7f9173ddd5af 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt @@ -1,96 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ^ InputAdapter (60) - +- ^ ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- ^ InputAdapter (18) - : : : +- ^ BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- ^ InputAdapter (27) - : : +- ^ BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- ^ InputAdapter (36) - : +- ^ BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (58) + +- ^ SortExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ^ InputAdapter (54) + +- ^ ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ BroadcastHashJoinExecTransformer Inner (41) + :- ^ ProjectExecTransformer (34) + : +- ^ BroadcastHashJoinExecTransformer Inner (33) + : :- ^ ProjectExecTransformer (26) + : : +- ^ BroadcastHashJoinExecTransformer Inner (25) + : : :- ^ ProjectExecTransformer (18) + : : : +- ^ BroadcastHashJoinExecTransformer Inner (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner (9) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ ProjectExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- ^ InputAdapter (15) + : : : +- ^ BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (24) + : : +- ^ InputAdapter (23) + : : +- ^ BroadcastQueryStage (22), Statistics(X) + : : +- ColumnarBroadcastExchange (21) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ BroadcastQueryStage (30), Statistics(X) + : +- ColumnarBroadcastExchange (29) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (40) + +- ^ InputAdapter (39) + +- ^ BroadcastQueryStage (38), Statistics(X) + +- ColumnarBroadcastExchange (37) + +- ^ Scan parquet (35) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (91) + +- Exchange (90) + +- HashAggregate (89) + +- Exchange (88) + +- HashAggregate (87) + +- Project (86) + +- BroadcastHashJoin Inner BuildRight (85) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (76) + : : +- BroadcastHashJoin Inner BuildRight (75) + : : :- Project (71) + : : : +- BroadcastHashJoin Inner BuildRight (70) + : : : :- Project (66) + : : : : +- BroadcastHashJoin Inner BuildLeft (65) + : : : : :- BroadcastExchange (62) + : : : : : +- Project (61) + : : : : : +- Filter (60) + : : : : : +- Scan parquet (59) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- BroadcastExchange (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- BroadcastExchange (79) + : +- Filter (78) + : +- Scan parquet (77) + +- BroadcastExchange (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -100,440 +94,416 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(5) ColumnarBroadcastExchange +(4) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(6) BroadcastQueryStage +(5) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(8) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) BroadcastHashJoinExecTransformer +(9) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(12) ProjectExecTransformer +(10) ProjectExecTransformer Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(13) Scan parquet +(11) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(14) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(15) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(13) ColumnarBroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(17) BroadcastQueryStage +(14) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(18) InputAdapter +(15) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(19) InputIteratorTransformer +(16) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(20) BroadcastHashJoinExecTransformer +(17) BroadcastHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(21) ProjectExecTransformer +(18) ProjectExecTransformer Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(22) Scan parquet +(19) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(23) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) - -(24) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(25) ColumnarBroadcastExchange +(21) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(26) BroadcastQueryStage +(22) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(27) InputAdapter +(23) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(28) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(29) BroadcastHashJoinExecTransformer +(25) BroadcastHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(30) ProjectExecTransformer +(26) ProjectExecTransformer Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(31) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(32) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: isnotnull(o_orderkey#X) - -(33) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: false -(34) ColumnarBroadcastExchange +(29) ColumnarBroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(35) BroadcastQueryStage +(30) BroadcastQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(36) InputAdapter +(31) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(37) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(38) BroadcastHashJoinExecTransformer +(33) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(39) ProjectExecTransformer +(34) ProjectExecTransformer Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(40) Scan parquet +(35) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(41) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(42) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarBroadcastExchange +(37) ColumnarBroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastQueryStage +(38) BroadcastQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(39) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(40) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) BroadcastHashJoinExecTransformer +(41) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(42) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(44) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(46) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(47) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(48) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(49) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(57) WholeStageCodegenTransformer (X) +(51) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(52) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(53) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(54) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(55) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(56) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(59) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(60) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(61) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(62) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(63) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(64) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(65) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(72) Project +(66) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(67) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(68) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(69) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(70) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(77) Project +(71) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(72) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(73) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(74) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(82) Project +(76) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(77) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(78) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(79) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(87) Project +(81) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(84) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(85) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(92) Project +(86) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(87) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(88) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(89) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(90) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(91) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt index 7a80ed95f3e8..699c17ea4562 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt @@ -1,31 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (27) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (18) + +- ^ SortExecTransformer (16) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13) + +- ColumnarExchange (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7) + +- ColumnarExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (3) + +- ^ ProjectExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (26) + +- Exchange (25) + +- HashAggregate (24) + +- Exchange (23) + +- HashAggregate (22) + +- Project (21) + +- Filter (20) + +- Scan parquet (19) (1) Scan parquet @@ -35,120 +34,116 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) FilterExecTransformer -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Arguments: (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(3) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) ProjectExecTransformer +(4) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(6) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(6) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(7) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(8) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(9) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(12) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(13) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(14) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(15) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(16) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(18) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(19) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(20) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(21) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(22) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(23) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(24) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(25) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(26) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(27) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt index 6b7f5bab6b1c..35c0d13f1e1e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt @@ -1,85 +1,81 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (83) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (56) + +- TakeOrderedAndProjectExecTransformer (55) + +- ^ ProjectExecTransformer (53) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ ShuffledHashJoinExecTransformer Inner (43) + :- ^ InputIteratorTransformer (35) + : +- ^ InputAdapter (34) + : +- ^ ShuffleQueryStage (33) + : +- ColumnarExchange (32) + : +- ^ ProjectExecTransformer (30) + : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : :- ^ InputIteratorTransformer (21) + : : +- ^ InputAdapter (20) + : : +- ^ ShuffleQueryStage (19) + : : +- ColumnarExchange (18) + : : +- ^ ProjectExecTransformer (16) + : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ ShuffleQueryStage (5) + : : : +- ColumnarExchange (4) + : : : +- ^ ProjectExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (14) + : : +- ^ InputAdapter (13) + : : +- ^ ShuffleQueryStage (12) + : : +- ColumnarExchange (11) + : : +- ^ ProjectExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (28) + : +- ^ InputAdapter (27) + : +- ^ ShuffleQueryStage (26) + : +- ColumnarExchange (25) + : +- ^ ProjectExecTransformer (23) + : +- ^ Scan parquet (22) + +- ^ InputIteratorTransformer (42) + +- ^ InputAdapter (41) + +- ^ ShuffleQueryStage (40) + +- ColumnarExchange (39) + +- ^ ProjectExecTransformer (37) + +- ^ Scan parquet (36) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (82) + +- HashAggregate (81) + +- Exchange (80) + +- HashAggregate (79) + +- Project (78) + +- ShuffledHashJoin Inner BuildRight (77) + :- Exchange (73) + : +- Project (72) + : +- ShuffledHashJoin Inner BuildRight (71) + : :- Exchange (66) + : : +- Project (65) + : : +- ShuffledHashJoin Inner BuildRight (64) + : : :- Exchange (59) + : : : +- Filter (58) + : : : +- Scan parquet (57) + : : +- Exchange (63) + : : +- Project (62) + : : +- Filter (61) + : : +- Scan parquet (60) + : +- Exchange (70) + : +- Project (69) + : +- Filter (68) + : +- Scan parquet (67) + +- Exchange (76) + +- Filter (75) + +- Scan parquet (74) (1) Scan parquet @@ -89,370 +85,354 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [8]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(22) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(36) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(46) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(48) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(49) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(50) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(51) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(57) ProjectExecTransformer +(53) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(58) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(55) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(57) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(58) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(59) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(60) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(61) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(62) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(63) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(64) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(65) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(66) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(67) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(68) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(69) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(70) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(71) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(72) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(73) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(74) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(75) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(76) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(77) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(78) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(79) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(80) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(81) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(82) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(83) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt index 4fe3c8b62ca9..23cd40b681e6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt @@ -1,71 +1,68 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ ShuffleQueryStage (38) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ^ InputAdapter (43) + +- ^ ShuffleQueryStage (42) + +- ColumnarExchange (41) + +- ^ FilterExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ^ InputAdapter (36) + +- ^ ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- ^ ProjectExecTransformer (32) + +- ^ FlushableHashAggregateExecTransformer (31) + +- ^ ProjectExecTransformer (30) + +- ^ ShuffledHashJoinExecTransformer Inner (29) + :- ^ InputIteratorTransformer (21) + : +- ^ InputAdapter (20) + : +- ^ ShuffleQueryStage (19) + : +- ColumnarExchange (18) + : +- ^ ProjectExecTransformer (16) + : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (14) + : +- ^ InputAdapter (13) + : +- ^ ShuffleQueryStage (12) + : +- ColumnarExchange (11) + : +- ^ ProjectExecTransformer (9) + : +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (28) + +- ^ InputAdapter (27) + +- ^ ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ Scan parquet (22) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (68) + +- Exchange (67) + +- Filter (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- ShuffledHashJoin Inner BuildRight (61) + :- Exchange (56) + : +- Project (55) + : +- ShuffledHashJoin Inner BuildRight (54) + : :- Exchange (50) + : : +- Filter (49) + : : +- Scan parquet (48) + : +- Exchange (53) + : +- Filter (52) + : +- Scan parquet (51) + +- Exchange (60) + +- Project (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -75,304 +72,292 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(8) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(22) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(31) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(34) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(35) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(36) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(37) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(42) FilterExecTransformer +(39) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(41) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(43) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(45) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(48) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(49) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(50) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(51) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(52) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(53) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(55) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(56) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(57) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(58) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(59) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(60) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(61) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(62) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(63) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(64) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(65) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(69) Filter +(66) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(67) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(68) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt index f8bca5a8c389..c5f7a17f4286 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt @@ -1,50 +1,48 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (47) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ RegularHashAggregateExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21) + +- ColumnarExchange (20) + +- ^ ProjectExecTransformer (18) + +- ^ FlushableHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer Inner (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (46) + +- Exchange (45) + +- HashAggregate (44) + +- Exchange (43) + +- HashAggregate (42) + +- Project (41) + +- ShuffledHashJoin Inner BuildLeft (40) + :- Exchange (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Exchange (39) + +- Project (38) + +- Filter (37) + +- Scan parquet (36) (1) Scan parquet @@ -54,208 +52,200 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: isnotnull(o_orderkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(17) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(24) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(26) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(28) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(30) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(33) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(34) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(35) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(36) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(37) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(38) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(39) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(40) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(41) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(43) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(44) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(45) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(46) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(47) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt index 14f2b770f996..507966beca53 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt @@ -1,53 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (51) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer LeftOuter (15) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5) : +- ColumnarExchange (4) : +- ^ ProjectExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (50) + +- Exchange (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftOuter BuildRight (42) + :- Exchange (37) + : +- Scan parquet (36) + +- Exchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -85,202 +84,198 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) - -(10) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(13) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(16) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(19) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(21) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(25) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(29) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(31) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(33) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(36) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(37) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(39) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(40) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(43) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(44) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(46) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(47) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(48) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(49) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(50) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(51) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt index 3be22928e4f2..fd6ee41dbfa9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt @@ -1,38 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (33) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (21) + +- ^ ProjectExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer Inner (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (32) + +- HashAggregate (31) + +- Project (30) + +- ShuffledHashJoin Inner BuildRight (29) + :- Exchange (25) + : +- Project (24) + : +- Filter (23) + : +- Scan parquet (22) + +- Exchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -42,156 +40,148 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(8) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: isnotnull(p_partkey#X) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(22) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(23) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(24) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(25) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(26) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(27) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(28) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(29) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(30) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(31) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(32) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(33) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt index 82b075711ecc..66d138fef6d1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt @@ -1,47 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner (21) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (28) + +- ^ SortExecTransformer (26) + +- ^ InputIteratorTransformer (25) + +- ^ InputAdapter (24) + +- ^ ShuffleQueryStage (23) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner (19) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FlushableHashAggregateExecTransformer (10) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- ShuffledHashJoin Inner BuildLeft (41) - :- Exchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (42) + +- Exchange (41) + +- Project (40) + +- ShuffledHashJoin Inner BuildLeft (39) + :- Exchange (31) + : +- Filter (30) + : +- Scan parquet (29) + +- Filter (38) + +- HashAggregate (37) + +- Exchange (36) + +- HashAggregate (35) + +- Project (34) + +- Filter (33) + +- Scan parquet (32) (1) Scan parquet @@ -51,194 +49,186 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: isnotnull(s_suppkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(10) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(13) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(14) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(15) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(16) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(20) FilterExecTransformer +(18) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(22) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(23) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) InputAdapter +(24) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) InputIteratorTransformer +(25) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) SortExecTransformer +(26) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(30) VeloxColumnarToRowExec +(28) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(31) Scan parquet +(29) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(32) Filter +(30) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(33) Exchange +(31) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Scan parquet +(32) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(33) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(34) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(35) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(36) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(40) Filter +(38) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) ShuffledHashJoin +(39) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(42) Project +(40) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(41) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(42) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt index 4eff981d78bd..8acd437dd729 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt @@ -1,64 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (62) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ^ InputAdapter (38) - +- ^ ShuffleQueryStage (37) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ^ InputAdapter (36) + +- ^ ShuffleQueryStage (35) + +- ColumnarExchange (34) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ FlushableHashAggregateExecTransformer (25) + +- ^ RegularHashAggregateExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21) + +- ColumnarExchange (20) + +- ^ ProjectExecTransformer (18) + +- ^ FlushableHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer Inner (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (61) + +- Exchange (60) + +- HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- ShuffledHashJoin Inner BuildRight (52) + :- Exchange (48) + : +- BroadcastHashJoin LeftAnti BuildRight (47) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Exchange (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -68,282 +66,274 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: isnotnull(ps_partkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Input [2]: [ps_partkey#X, ps_suppkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(17) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(24) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(25) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(26) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(28) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(29) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(30) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(31) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(34) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(38) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(41) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(42) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(44) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(45) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(46) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(48) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(49) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(50) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(51) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(52) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(53) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(54) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(57) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(58) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(59) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(61) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(62) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt index 04f31f2d7a90..a759dc4f95c1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt @@ -1,59 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ ProjectExecTransformer (29) + +- ^ ShuffledHashJoinExecTransformer Inner (28) + :- ^ ProjectExecTransformer (16) + : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (14) + : +- ^ InputAdapter (13) + : +- ^ ShuffleQueryStage (12) + : +- ColumnarExchange (11) + : +- ^ ProjectExecTransformer (9) + : +- ^ Scan parquet (8) + +- ^ FilterExecTransformer (27) + +- ^ ProjectExecTransformer (26) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ Scan parquet (17) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (53) + +- HashAggregate (52) + +- Project (51) + +- ShuffledHashJoin Inner BuildRight (50) + :- Project (43) + : +- ShuffledHashJoin Inner BuildRight (42) + : :- Exchange (37) + : : +- Filter (36) + : : +- Scan parquet (35) + : +- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Filter (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -63,262 +60,250 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Arguments: (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [p_partkey#X, p_brand#X, p_container#X] -Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(17) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) FilterExecTransformer -Input [2]: [l_partkey#X, l_quantity#X] -Arguments: isnotnull(l_partkey#X) - -(21) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(19) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(21) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(23) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(27) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(32) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(35) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(36) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(37) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(38) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(39) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(40) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(41) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(43) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(44) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(45) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(46) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(47) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(48) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(49) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(50) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(51) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(52) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(53) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt index ac59fbf6e10f..7a6e11258914 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt @@ -1,96 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) - :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) - :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51) - +- ReusedExchange (50) + VeloxColumnarToRowExec (61) + +- TakeOrderedAndProjectExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ ShuffledHashJoinExecTransformer Inner (55) + :- ^ InputIteratorTransformer (39) + : +- ^ InputAdapter (38) + : +- ^ ShuffleQueryStage (37) + : +- ColumnarExchange (36) + : +- ^ ProjectExecTransformer (34) + : +- ^ ShuffledHashJoinExecTransformer Inner (33) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ ShuffleQueryStage (30) + : +- ColumnarExchange (29) + : +- ^ ProjectExecTransformer (27) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (26) + : :- ^ InputIteratorTransformer (14) + : : +- ^ InputAdapter (13) + : : +- ^ ShuffleQueryStage (12) + : : +- ColumnarExchange (11) + : : +- ^ ProjectExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ ProjectExecTransformer (25) + : +- ^ FilterExecTransformer (24) + : +- ^ RegularHashAggregateExecTransformer (23) + : +- ^ InputIteratorTransformer (22) + : +- ^ InputAdapter (21) + : +- ^ ShuffleQueryStage (20) + : +- ColumnarExchange (19) + : +- ^ ProjectExecTransformer (17) + : +- ^ FlushableHashAggregateExecTransformer (16) + : +- ^ Scan parquet (15) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (54) + :- ^ InputIteratorTransformer (46) + : +- ^ InputAdapter (45) + : +- ^ ShuffleQueryStage (44) + : +- ColumnarExchange (43) + : +- ^ ProjectExecTransformer (41) + : +- ^ Scan parquet (40) + +- ^ ProjectExecTransformer (53) + +- ^ FilterExecTransformer (52) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ^ InputAdapter (49) + +- ^ ShuffleQueryStage (48) + +- ReusedExchange (47) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- HashAggregate (91) + +- Project (90) + +- ShuffledHashJoin Inner BuildRight (89) + :- Exchange (78) + : +- Project (77) + : +- ShuffledHashJoin Inner BuildLeft (76) + : :- Exchange (64) + : : +- Filter (63) + : : +- Scan parquet (62) + : +- Exchange (75) + : +- ShuffledHashJoin LeftSemi BuildRight (74) + : :- Exchange (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Project (73) + : +- Filter (72) + : +- HashAggregate (71) + : +- Exchange (70) + : +- HashAggregate (69) + : +- Scan parquet (68) + +- ShuffledHashJoin LeftSemi BuildRight (88) + :- Exchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- Project (87) + +- Filter (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Scan parquet (82) (1) Scan parquet @@ -100,432 +97,420 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_name#X] -Arguments: isnotnull(c_custkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X] Input [2]: [c_custkey#X, c_name#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(15) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(16) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(17) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(19) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(20) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(21) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(22) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(24) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(25) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(26) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(27) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(29) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(30) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(31) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(32) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(34) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(36) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(37) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(38) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(39) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(40) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) FilterExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: isnotnull(l_orderkey#X) - -(44) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(45) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(47) ReusedExchange [Reuses operator id: 19] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(48) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(49) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(50) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(52) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(54) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(56) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(60) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(62) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(63) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(64) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(65) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(66) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(67) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(68) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(69) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(70) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(71) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(72) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(73) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(76) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(77) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(78) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(79) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(80) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(81) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(82) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(83) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(84) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(85) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(86) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(87) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(89) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(90) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(91) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(92) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt index 4bff7da9b473..260269c9eaef 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt @@ -1,37 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (32) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer Inner (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (31) + +- HashAggregate (30) + +- Project (29) + +- ShuffledHashJoin Inner BuildRight (28) + :- Exchange (24) + : +- Project (23) + : +- Filter (22) + : +- Scan parquet (21) + +- Exchange (27) + +- Filter (26) + +- Scan parquet (25) (1) Scan parquet @@ -41,152 +39,144 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) FilterExecTransformer -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Arguments: ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(21) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(22) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(23) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(24) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(25) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(26) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(27) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(28) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(29) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(30) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(31) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(34) AdaptiveSparkPlan +(32) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt index 15ea12dd248e..2071bbf73f12 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt @@ -1,121 +1,116 @@ == Physical Plan == -AdaptiveSparkPlan (126) +AdaptiveSparkPlan (121) +- == Final Plan == - VeloxColumnarToRowExec (86) - +- ^ SortExecTransformer (84) - +- ^ InputIteratorTransformer (83) - +- ^ InputAdapter (82) - +- ^ ShuffleQueryStage (81) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) - : :- ^ InputIteratorTransformer (31) - : : +- ^ InputAdapter (30) - : : +- ^ ShuffleQueryStage (29) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ ShuffleQueryStage (14) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ ShuffleQueryStage (22) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) - : :- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ^ InputAdapter (42) - : +- ^ ShuffleQueryStage (41) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (81) + +- ^ SortExecTransformer (79) + +- ^ InputIteratorTransformer (78) + +- ^ InputAdapter (77) + +- ^ ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ ShuffledHashJoinExecTransformer Inner (72) + :- ^ InputIteratorTransformer (64) + : +- ^ InputAdapter (63) + : +- ^ ShuffleQueryStage (62) + : +- ColumnarExchange (61) + : +- ^ ProjectExecTransformer (59) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (58) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (57) + : +- ^ InputAdapter (56) + : +- ^ ShuffleQueryStage (55) + : +- ColumnarExchange (54) + : +- ^ ProjectExecTransformer (52) + : +- ^ ShuffledHashJoinExecTransformer Inner (51) + : :- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ ShuffleQueryStage (26) + : : +- ColumnarExchange (25) + : : +- ^ ProjectExecTransformer (23) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (22) + : : :- ^ InputIteratorTransformer (14) + : : : +- ^ InputAdapter (13) + : : : +- ^ ShuffleQueryStage (12) + : : : +- ColumnarExchange (11) + : : : +- ^ ProjectExecTransformer (9) + : : : +- ^ Scan parquet (8) + : : +- ^ InputIteratorTransformer (21) + : : +- ^ InputAdapter (20) + : : +- ^ ShuffleQueryStage (19) + : : +- ColumnarExchange (18) + : : +- ^ ProjectExecTransformer (16) + : : +- ^ Scan parquet (15) + : +- ^ InputIteratorTransformer (50) + : +- ^ InputAdapter (49) + : +- ^ ShuffleQueryStage (48) + : +- ColumnarExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ FilterExecTransformer (44) + : +- ^ ProjectExecTransformer (43) + : +- ^ RegularHashAggregateExecTransformer (42) + : +- ^ RegularHashAggregateExecTransformer (41) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (40) + : :- ^ InputIteratorTransformer (35) + : : +- ^ InputAdapter (34) + : : +- ^ ShuffleQueryStage (33) + : : +- ColumnarExchange (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ Scan parquet (29) + : +- ^ InputIteratorTransformer (39) + : +- ^ InputAdapter (38) + : +- ^ ShuffleQueryStage (37) + : +- ReusedExchange (36) + +- ^ InputIteratorTransformer (71) + +- ^ InputAdapter (70) + +- ^ ShuffleQueryStage (69) + +- ColumnarExchange (68) + +- ^ ProjectExecTransformer (66) + +- ^ Scan parquet (65) +- == Initial Plan == - Sort (125) - +- Exchange (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin LeftSemi BuildRight (115) - : :- Exchange (89) - : : +- Filter (88) - : : +- Scan parquet (87) - : +- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin Inner BuildLeft (112) - : :- Exchange (98) - : : +- ShuffledHashJoin LeftSemi BuildRight (97) - : : :- Exchange (92) - : : : +- Filter (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (111) - : +- Filter (110) - : +- HashAggregate (109) - : +- HashAggregate (108) - : +- ShuffledHashJoin LeftSemi BuildRight (107) - : :- Exchange (102) - : : +- Project (101) - : : +- Filter (100) - : : +- Scan parquet (99) - : +- Exchange (106) - : +- Project (105) - : +- Filter (104) - : +- Scan parquet (103) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (120) + +- Exchange (119) + +- Project (118) + +- ShuffledHashJoin Inner BuildRight (117) + :- Exchange (112) + : +- Project (111) + : +- ShuffledHashJoin LeftSemi BuildRight (110) + : :- Exchange (84) + : : +- Filter (83) + : : +- Scan parquet (82) + : +- Exchange (109) + : +- Project (108) + : +- ShuffledHashJoin Inner BuildLeft (107) + : :- Exchange (93) + : : +- ShuffledHashJoin LeftSemi BuildRight (92) + : : :- Exchange (87) + : : : +- Filter (86) + : : : +- Scan parquet (85) + : : +- Exchange (91) + : : +- Project (90) + : : +- Filter (89) + : : +- Scan parquet (88) + : +- Exchange (106) + : +- Filter (105) + : +- HashAggregate (104) + : +- HashAggregate (103) + : +- ShuffledHashJoin LeftSemi BuildRight (102) + : :- Exchange (97) + : : +- Project (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (101) + : +- Project (100) + : +- Filter (99) + : +- Scan parquet (98) + +- Exchange (116) + +- Project (115) + +- Filter (114) + +- Scan parquet (113) (1) Scan parquet @@ -125,530 +120,510 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: isnotnull(s_nationkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(15) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) - -(19) ProjectExecTransformer +(16) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(18) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(19) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(20) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(21) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(29) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) FilterExecTransformer -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) - -(34) ProjectExecTransformer +(30) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(34) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(36) ReusedExchange [Reuses operator id: 18] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(37) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(38) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(39) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(40) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(42) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(43) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(44) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(47) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(48) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(49) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(50) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(51) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(52) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(54) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(55) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(56) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(57) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(58) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(59) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(61) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(62) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(63) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(64) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) - -(71) ProjectExecTransformer +(66) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(68) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(69) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(70) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(71) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(72) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(73) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(75) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) InputAdapter +(77) InputAdapter Input [2]: [s_name#X, s_address#X] -(83) InputIteratorTransformer +(78) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(84) SortExecTransformer +(79) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(85) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(86) VeloxColumnarToRowExec +(81) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(87) Scan parquet +(82) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(88) Filter +(83) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(89) Exchange +(84) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(85) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(91) Filter +(86) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(92) Exchange +(87) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(88) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(94) Filter +(89) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(95) Project +(90) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(96) Exchange +(91) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(98) Exchange +(93) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(94) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(95) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(101) Project +(96) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(102) Exchange +(97) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(98) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(104) Filter +(99) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(105) Project +(100) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(106) Exchange +(101) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(108) HashAggregate +(103) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(109) HashAggregate +(104) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(110) Filter +(105) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(111) Exchange +(106) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(113) Project +(108) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(114) Exchange +(109) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(110) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(116) Project +(111) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(117) Exchange +(112) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(113) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(114) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(120) Project +(115) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(116) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(123) Project +(118) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(124) Exchange +(119) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(120) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(126) AdaptiveSparkPlan +(121) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt index 17fad29860df..c4f7c527a056 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt @@ -1,114 +1,109 @@ == Physical Plan == -AdaptiveSparkPlan (119) +AdaptiveSparkPlan (114) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- TakeOrderedAndProjectExecTransformer (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner (70) - :- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) - : :- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ^ InputAdapter (22) - : : : +- ^ ShuffleQueryStage (21) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ^ InputAdapter (31) - : : +- ^ ShuffleQueryStage (30) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ^ InputAdapter (68) - +- ^ ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (77) + +- TakeOrderedAndProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (74) + +- ^ InputIteratorTransformer (73) + +- ^ InputAdapter (72) + +- ^ ShuffleQueryStage (71) + +- ColumnarExchange (70) + +- ^ ProjectExecTransformer (68) + +- ^ FlushableHashAggregateExecTransformer (67) + +- ^ ProjectExecTransformer (66) + +- ^ ShuffledHashJoinExecTransformer Inner (65) + :- ^ InputIteratorTransformer (57) + : +- ^ InputAdapter (56) + : +- ^ ShuffleQueryStage (55) + : +- ColumnarExchange (54) + : +- ^ ProjectExecTransformer (52) + : +- ^ ShuffledHashJoinExecTransformer Inner (51) + : :- ^ InputIteratorTransformer (43) + : : +- ^ InputAdapter (42) + : : +- ^ ShuffleQueryStage (41) + : : +- ColumnarExchange (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer Inner (37) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ ShuffleQueryStage (5) + : : : +- ColumnarExchange (4) + : : : +- ^ ProjectExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (36) + : : +- ^ InputAdapter (35) + : : +- ^ ShuffleQueryStage (34) + : : +- ColumnarExchange (33) + : : +- ^ ProjectExecTransformer (31) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (30) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (22) + : : : :- ^ InputIteratorTransformer (14) + : : : : +- ^ InputAdapter (13) + : : : : +- ^ ShuffleQueryStage (12) + : : : : +- ColumnarExchange (11) + : : : : +- ^ ProjectExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (21) + : : : +- ^ InputAdapter (20) + : : : +- ^ ShuffleQueryStage (19) + : : : +- ColumnarExchange (18) + : : : +- ^ ProjectExecTransformer (16) + : : : +- ^ Scan parquet (15) + : : +- ^ InputIteratorTransformer (29) + : : +- ^ InputAdapter (28) + : : +- ^ ShuffleQueryStage (27) + : : +- ColumnarExchange (26) + : : +- ^ ProjectExecTransformer (24) + : : +- ^ Scan parquet (23) + : +- ^ InputIteratorTransformer (50) + : +- ^ InputAdapter (49) + : +- ^ ShuffleQueryStage (48) + : +- ColumnarExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ Scan parquet (44) + +- ^ InputIteratorTransformer (64) + +- ^ InputAdapter (63) + +- ^ ShuffleQueryStage (62) + +- ColumnarExchange (61) + +- ^ ProjectExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - TakeOrderedAndProject (118) - +- HashAggregate (117) - +- Exchange (116) - +- HashAggregate (115) - +- Project (114) - +- ShuffledHashJoin Inner BuildRight (113) - :- Exchange (108) - : +- Project (107) - : +- ShuffledHashJoin Inner BuildRight (106) - : :- Exchange (101) - : : +- Project (100) - : : +- ShuffledHashJoin Inner BuildLeft (99) - : : :- Exchange (85) - : : : +- Filter (84) - : : : +- Scan parquet (83) - : : +- Exchange (98) - : : +- ShuffledHashJoin LeftAnti BuildRight (97) - : : :- ShuffledHashJoin LeftSemi BuildRight (92) - : : : :- Exchange (89) - : : : : +- Project (88) - : : : : +- Filter (87) - : : : : +- Scan parquet (86) - : : : +- Exchange (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (105) - : +- Project (104) - : +- Filter (103) - : +- Scan parquet (102) - +- Exchange (112) - +- Project (111) - +- Filter (110) - +- Scan parquet (109) + TakeOrderedAndProject (113) + +- HashAggregate (112) + +- Exchange (111) + +- HashAggregate (110) + +- Project (109) + +- ShuffledHashJoin Inner BuildRight (108) + :- Exchange (103) + : +- Project (102) + : +- ShuffledHashJoin Inner BuildRight (101) + : :- Exchange (96) + : : +- Project (95) + : : +- ShuffledHashJoin Inner BuildLeft (94) + : : :- Exchange (80) + : : : +- Filter (79) + : : : +- Scan parquet (78) + : : +- Exchange (93) + : : +- ShuffledHashJoin LeftAnti BuildRight (92) + : : :- ShuffledHashJoin LeftSemi BuildRight (87) + : : : :- Exchange (84) + : : : : +- Project (83) + : : : : +- Filter (82) + : : : : +- Scan parquet (81) + : : : +- Exchange (86) + : : : +- Scan parquet (85) + : : +- Exchange (91) + : : +- Project (90) + : : +- Filter (89) + : : +- Scan parquet (88) + : +- Exchange (100) + : +- Project (99) + : +- Filter (98) + : +- Scan parquet (97) + +- Exchange (107) + +- Project (106) + +- Filter (105) + +- Scan parquet (104) (1) Scan parquet @@ -118,506 +113,486 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(15) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(23) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) - -(27) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(28) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(31) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(40) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(41) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(42) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(43) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(44) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderstatus#X] -Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) - -(49) ProjectExecTransformer +(45) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(47) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(48) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(49) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(50) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(51) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(54) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(55) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(56) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(57) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(58) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) - -(64) ProjectExecTransformer +(59) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(61) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(62) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(63) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(64) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(65) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(66) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(67) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(69) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(70) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(71) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(72) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(73) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(74) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) TakeOrderedAndProjectExecTransformer +(76) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(82) VeloxColumnarToRowExec +(77) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(83) Scan parquet +(78) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(84) Filter +(79) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(85) Exchange +(80) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) Scan parquet +(81) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(87) Filter +(82) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(88) Project +(83) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(89) Exchange +(84) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(85) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) Exchange +(86) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(93) Scan parquet +(88) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(94) Filter +(89) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(95) Project +(90) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(96) Exchange +(91) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(98) Exchange +(93) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(94) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(100) Project +(95) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(101) Exchange +(96) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Scan parquet +(97) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(103) Filter +(98) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(104) Project +(99) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(105) Exchange +(100) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(107) Project +(102) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(108) Exchange +(103) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(104) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(110) Filter +(105) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(111) Project +(106) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(112) Exchange +(107) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(114) Project +(109) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(115) HashAggregate +(110) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(116) Exchange +(111) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) HashAggregate +(112) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(118) TakeOrderedAndProject +(113) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(119) AdaptiveSparkPlan +(114) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt index 918335dff38b..0f13ad3079c9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt @@ -1,47 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ^ InputAdapter (29) - +- ^ ShuffleQueryStage (28) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ RegularHashAggregateExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21) + +- ColumnarExchange (20) + +- ^ ProjectExecTransformer (18) + +- ^ FlushableHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer LeftAnti (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- ShuffledHashJoin LeftAnti BuildRight (38) + :- Exchange (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Exchange (37) + +- Scan parquet (36) (1) Scan parquet @@ -51,194 +50,190 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(8) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(9) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(11) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(12) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(13) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(14) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(16) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(17) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(22) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(24) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(26) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(28) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(30) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(33) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(34) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(35) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(36) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(37) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(38) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(39) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(40) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(41) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(42) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(43) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(44) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt index 157e13d105e4..318b9bdf6a46 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt @@ -1,60 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (36) + +- TakeOrderedAndProjectExecTransformer (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ ProjectExecTransformer (30) + +- ^ ShuffledHashJoinExecTransformer Inner (29) + :- ^ InputIteratorTransformer (21) + : +- ^ InputAdapter (20) + : +- ^ ShuffleQueryStage (19) + : +- ColumnarExchange (18) + : +- ^ ProjectExecTransformer (16) + : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (14) + : +- ^ InputAdapter (13) + : +- ^ ShuffleQueryStage (12) + : +- ColumnarExchange (11) + : +- ^ ProjectExecTransformer (9) + : +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (28) + +- ^ InputAdapter (27) + +- ^ ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ Scan parquet (22) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Project (52) + +- ShuffledHashJoin Inner BuildRight (51) + :- Exchange (46) + : +- Project (45) + : +- ShuffledHashJoin Inner BuildLeft (44) + : :- Exchange (40) + : : +- Project (39) + : : +- Filter (38) + : : +- Scan parquet (37) + : +- Exchange (43) + : +- Filter (42) + : +- Scan parquet (41) + +- Exchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -64,256 +61,244 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_mktsegment#X] -Arguments: ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(22) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(36) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(37) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(35) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(37) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(38) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(39) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(40) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(41) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(42) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(43) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(44) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(45) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(46) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(47) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(48) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(49) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(50) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(52) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(53) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(54) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(55) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt index 74fca4208dd5..807c29568ead 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt @@ -1,51 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (48) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ RegularHashAggregateExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21) + +- ColumnarExchange (20) + +- ^ ProjectExecTransformer (18) + +- ^ FlushableHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (47) + +- Exchange (46) + +- HashAggregate (45) + +- Exchange (44) + +- HashAggregate (43) + +- Project (42) + +- ShuffledHashJoin LeftSemi BuildRight (41) + :- Exchange (36) + : +- Project (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Exchange (40) + +- Project (39) + +- Filter (38) + +- Scan parquet (37) (1) Scan parquet @@ -55,212 +53,204 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Arguments: ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(17) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(24) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(26) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(28) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(30) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(33) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(34) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(35) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(36) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(37) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(38) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(39) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(40) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(42) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(43) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(45) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(46) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(47) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(48) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt index c01060f99d4b..fc394df77026 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt @@ -1,127 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (88) + +- ^ SortExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ^ InputAdapter (78) + +- ^ ShuffleQueryStage (77) + +- ColumnarExchange (76) + +- ^ ProjectExecTransformer (74) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ ShuffledHashJoinExecTransformer Inner (71) + :- ^ InputIteratorTransformer (63) + : +- ^ InputAdapter (62) + : +- ^ ShuffleQueryStage (61) + : +- ColumnarExchange (60) + : +- ^ ProjectExecTransformer (58) + : +- ^ ShuffledHashJoinExecTransformer Inner (57) + : :- ^ InputIteratorTransformer (49) + : : +- ^ InputAdapter (48) + : : +- ^ ShuffleQueryStage (47) + : : +- ColumnarExchange (46) + : : +- ^ ProjectExecTransformer (44) + : : +- ^ ShuffledHashJoinExecTransformer Inner (43) + : : :- ^ InputIteratorTransformer (35) + : : : +- ^ InputAdapter (34) + : : : +- ^ ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- ^ ProjectExecTransformer (30) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : : : :- ^ InputIteratorTransformer (21) + : : : : +- ^ InputAdapter (20) + : : : : +- ^ ShuffleQueryStage (19) + : : : : +- ColumnarExchange (18) + : : : : +- ^ ProjectExecTransformer (16) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ ShuffleQueryStage (5) + : : : : : +- ColumnarExchange (4) + : : : : : +- ^ ProjectExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (14) + : : : : +- ^ InputAdapter (13) + : : : : +- ^ ShuffleQueryStage (12) + : : : : +- ColumnarExchange (11) + : : : : +- ^ ProjectExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (28) + : : : +- ^ InputAdapter (27) + : : : +- ^ ShuffleQueryStage (26) + : : : +- ColumnarExchange (25) + : : : +- ^ ProjectExecTransformer (23) + : : : +- ^ Scan parquet (22) + : : +- ^ InputIteratorTransformer (42) + : : +- ^ InputAdapter (41) + : : +- ^ ShuffleQueryStage (40) + : : +- ColumnarExchange (39) + : : +- ^ ProjectExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (56) + : +- ^ InputAdapter (55) + : +- ^ ShuffleQueryStage (54) + : +- ColumnarExchange (53) + : +- ^ ProjectExecTransformer (51) + : +- ^ Scan parquet (50) + +- ^ InputIteratorTransformer (70) + +- ^ InputAdapter (69) + +- ^ ShuffleQueryStage (68) + +- ColumnarExchange (67) + +- ^ ProjectExecTransformer (65) + +- ^ Scan parquet (64) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (127) + +- Exchange (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildRight (108) + : : :- Exchange (104) + : : : +- Project (103) + : : : +- ShuffledHashJoin Inner BuildRight (102) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- ShuffledHashJoin Inner BuildLeft (96) + : : : : :- Exchange (91) + : : : : : +- Filter (90) + : : : : : +- Scan parquet (89) + : : : : +- Exchange (95) + : : : : +- Project (94) + : : : : +- Filter (93) + : : : : +- Scan parquet (92) + : : : +- Exchange (101) + : : : +- Filter (100) + : : : +- Scan parquet (99) + : : +- Exchange (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Exchange (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -131,564 +125,540 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(22) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(36) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(46) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(47) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(48) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(49) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(50) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) FilterExecTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(56) ProjectExecTransformer +(51) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(53) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(55) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(60) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(61) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(62) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(63) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(64) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) - -(71) ProjectExecTransformer +(65) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(67) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(68) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(69) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(70) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(71) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(72) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(73) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(74) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(76) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(77) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(78) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(79) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(82) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(83) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(84) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(85) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(86) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(88) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(89) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(90) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(91) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(92) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(93) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(94) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(95) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(97) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(98) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(99) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(100) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(101) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(103) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(104) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(105) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(106) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(107) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(109) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(110) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(111) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(112) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(113) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(115) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(116) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(117) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(118) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(119) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(120) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(122) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(123) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(124) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(125) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(132) Exchange +(126) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(127) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt index fa2536a365c8..cc87fe6d2bbf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt @@ -1,23 +1,22 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (18) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (11) + +- ^ RegularHashAggregateExecTransformer (9) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ ShuffleQueryStage (6) + +- ColumnarExchange (5) + +- ^ FlushableHashAggregateExecTransformer (3) + +- ^ ProjectExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (17) + +- Exchange (16) + +- HashAggregate (15) + +- Project (14) + +- Filter (13) + +- Scan parquet (12) (1) Scan parquet @@ -27,86 +26,82 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(3) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(5) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(5) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(7) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(9) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(11) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(12) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(13) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(14) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(15) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(16) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(17) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(18) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt index 9cf916718039..b9b72a9de06a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt @@ -1,122 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (123) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ^ InputAdapter (86) - +- ^ ShuffleQueryStage (85) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner (73) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ^ InputAdapter (71) - +- ^ ShuffleQueryStage (70) - +- ReusedExchange (69) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ^ InputAdapter (81) + +- ^ ShuffleQueryStage (80) + +- ColumnarExchange (79) + +- ^ RegularHashAggregateExecTransformer (77) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ FlushableHashAggregateExecTransformer (70) + +- ^ ProjectExecTransformer (69) + +- ^ ShuffledHashJoinExecTransformer Inner (68) + :- ^ InputIteratorTransformer (63) + : +- ^ InputAdapter (62) + : +- ^ ShuffleQueryStage (61) + : +- ColumnarExchange (60) + : +- ^ ProjectExecTransformer (58) + : +- ^ ShuffledHashJoinExecTransformer Inner (57) + : :- ^ InputIteratorTransformer (49) + : : +- ^ InputAdapter (48) + : : +- ^ ShuffleQueryStage (47) + : : +- ColumnarExchange (46) + : : +- ^ ProjectExecTransformer (44) + : : +- ^ ShuffledHashJoinExecTransformer Inner (43) + : : :- ^ InputIteratorTransformer (35) + : : : +- ^ InputAdapter (34) + : : : +- ^ ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- ^ ProjectExecTransformer (30) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : : : :- ^ InputIteratorTransformer (21) + : : : : +- ^ InputAdapter (20) + : : : : +- ^ ShuffleQueryStage (19) + : : : : +- ColumnarExchange (18) + : : : : +- ^ ProjectExecTransformer (16) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ ShuffleQueryStage (5) + : : : : : +- ColumnarExchange (4) + : : : : : +- ^ ProjectExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (14) + : : : : +- ^ InputAdapter (13) + : : : : +- ^ ShuffleQueryStage (12) + : : : : +- ColumnarExchange (11) + : : : : +- ^ ProjectExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (28) + : : : +- ^ InputAdapter (27) + : : : +- ^ ShuffleQueryStage (26) + : : : +- ColumnarExchange (25) + : : : +- ^ ProjectExecTransformer (23) + : : : +- ^ Scan parquet (22) + : : +- ^ InputIteratorTransformer (42) + : : +- ^ InputAdapter (41) + : : +- ^ ShuffleQueryStage (40) + : : +- ColumnarExchange (39) + : : +- ^ ProjectExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (56) + : +- ^ InputAdapter (55) + : +- ^ ShuffleQueryStage (54) + : +- ColumnarExchange (53) + : +- ^ ProjectExecTransformer (51) + : +- ^ Scan parquet (50) + +- ^ InputIteratorTransformer (67) + +- ^ InputAdapter (66) + +- ^ ShuffleQueryStage (65) + +- ReusedExchange (64) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (122) + +- Exchange (121) + +- HashAggregate (120) + +- Exchange (119) + +- HashAggregate (118) + +- Project (117) + +- ShuffledHashJoin Inner BuildRight (116) + :- Exchange (112) + : +- Project (111) + : +- ShuffledHashJoin Inner BuildRight (110) + : :- Exchange (106) + : : +- Project (105) + : : +- ShuffledHashJoin Inner BuildRight (104) + : : :- Exchange (100) + : : : +- Project (99) + : : : +- ShuffledHashJoin Inner BuildRight (98) + : : : :- Exchange (94) + : : : : +- Project (93) + : : : : +- ShuffledHashJoin Inner BuildLeft (92) + : : : : :- Exchange (88) + : : : : : +- Filter (87) + : : : : : +- Scan parquet (86) + : : : : +- Exchange (91) + : : : : +- Filter (90) + : : : : +- Scan parquet (89) + : : : +- Exchange (97) + : : : +- Filter (96) + : : : +- Scan parquet (95) + : : +- Exchange (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Exchange (109) + : +- Filter (108) + : +- Scan parquet (107) + +- Exchange (115) + +- Filter (114) + +- Scan parquet (113) (1) Scan parquet @@ -126,536 +121,516 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(22) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(46) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(47) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(48) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(49) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(50) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) - -(56) ProjectExecTransformer +(51) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(53) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(55) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(60) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(61) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(62) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(63) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(64) ReusedExchange [Reuses operator id: 53] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(65) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(66) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(67) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(68) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(69) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(70) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(73) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(74) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(75) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(76) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(77) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(79) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(80) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(81) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(82) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(83) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(86) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(87) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(88) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(89) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(90) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(91) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(93) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(94) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(95) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(96) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(97) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(99) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(100) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(101) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(102) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(103) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(105) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(106) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(107) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(108) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(109) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(110) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(111) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(112) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(113) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(114) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(115) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(117) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(118) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(119) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(120) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(121) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(122) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(123) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt index 1e191ae4d23c..38e775dcd2ef 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt @@ -1,166 +1,158 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (169) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ^ InputAdapter (121) - +- ^ ShuffleQueryStage (120) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ^ InputAdapter (114) - +- ^ ShuffleQueryStage (113) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner (107) - :- ^ InputIteratorTransformer (98) - : +- ^ InputAdapter (97) - : +- ^ ShuffleQueryStage (96) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner (92) - : :- ^ InputIteratorTransformer (83) - : : +- ^ InputAdapter (82) - : : +- ^ ShuffleQueryStage (81) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ^ InputAdapter (67) - : : : +- ^ ShuffleQueryStage (66) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ^ InputAdapter (52) - : : : : +- ^ ShuffleQueryStage (51) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ^ InputAdapter (37) - : : : : : +- ^ ShuffleQueryStage (36) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ^ InputAdapter (22) - : : : : : : +- ^ ShuffleQueryStage (21) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ ShuffleQueryStage (6) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ^ InputAdapter (15) - : : : : : : +- ^ ShuffleQueryStage (14) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ^ InputAdapter (30) - : : : : : +- ^ ShuffleQueryStage (29) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ^ InputAdapter (45) - : : : : +- ^ ShuffleQueryStage (44) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ^ InputAdapter (60) - : : : +- ^ ShuffleQueryStage (59) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ^ InputAdapter (105) - +- ^ ShuffleQueryStage (104) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (117) + +- ^ SortExecTransformer (115) + +- ^ InputIteratorTransformer (114) + +- ^ InputAdapter (113) + +- ^ ShuffleQueryStage (112) + +- ColumnarExchange (111) + +- ^ ProjectExecTransformer (109) + +- ^ RegularHashAggregateExecTransformer (108) + +- ^ InputIteratorTransformer (107) + +- ^ InputAdapter (106) + +- ^ ShuffleQueryStage (105) + +- ColumnarExchange (104) + +- ^ ProjectExecTransformer (102) + +- ^ FlushableHashAggregateExecTransformer (101) + +- ^ ProjectExecTransformer (100) + +- ^ ShuffledHashJoinExecTransformer Inner (99) + :- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ ShuffledHashJoinExecTransformer Inner (85) + : :- ^ InputIteratorTransformer (77) + : : +- ^ InputAdapter (76) + : : +- ^ ShuffleQueryStage (75) + : : +- ColumnarExchange (74) + : : +- ^ ProjectExecTransformer (72) + : : +- ^ ShuffledHashJoinExecTransformer Inner (71) + : : :- ^ InputIteratorTransformer (63) + : : : +- ^ InputAdapter (62) + : : : +- ^ ShuffleQueryStage (61) + : : : +- ColumnarExchange (60) + : : : +- ^ ProjectExecTransformer (58) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (57) + : : : :- ^ InputIteratorTransformer (49) + : : : : +- ^ InputAdapter (48) + : : : : +- ^ ShuffleQueryStage (47) + : : : : +- ColumnarExchange (46) + : : : : +- ^ ProjectExecTransformer (44) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (43) + : : : : :- ^ InputIteratorTransformer (35) + : : : : : +- ^ InputAdapter (34) + : : : : : +- ^ ShuffleQueryStage (33) + : : : : : +- ColumnarExchange (32) + : : : : : +- ^ ProjectExecTransformer (30) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : : : : : :- ^ InputIteratorTransformer (21) + : : : : : : +- ^ InputAdapter (20) + : : : : : : +- ^ ShuffleQueryStage (19) + : : : : : : +- ColumnarExchange (18) + : : : : : : +- ^ ProjectExecTransformer (16) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : : : : : :- ^ InputIteratorTransformer (7) + : : : : : : : +- ^ InputAdapter (6) + : : : : : : : +- ^ ShuffleQueryStage (5) + : : : : : : : +- ColumnarExchange (4) + : : : : : : : +- ^ ProjectExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (14) + : : : : : : +- ^ InputAdapter (13) + : : : : : : +- ^ ShuffleQueryStage (12) + : : : : : : +- ColumnarExchange (11) + : : : : : : +- ^ ProjectExecTransformer (9) + : : : : : : +- ^ Scan parquet (8) + : : : : : +- ^ InputIteratorTransformer (28) + : : : : : +- ^ InputAdapter (27) + : : : : : +- ^ ShuffleQueryStage (26) + : : : : : +- ColumnarExchange (25) + : : : : : +- ^ ProjectExecTransformer (23) + : : : : : +- ^ Scan parquet (22) + : : : : +- ^ InputIteratorTransformer (42) + : : : : +- ^ InputAdapter (41) + : : : : +- ^ ShuffleQueryStage (40) + : : : : +- ColumnarExchange (39) + : : : : +- ^ ProjectExecTransformer (37) + : : : : +- ^ Scan parquet (36) + : : : +- ^ InputIteratorTransformer (56) + : : : +- ^ InputAdapter (55) + : : : +- ^ ShuffleQueryStage (54) + : : : +- ColumnarExchange (53) + : : : +- ^ ProjectExecTransformer (51) + : : : +- ^ Scan parquet (50) + : : +- ^ InputIteratorTransformer (70) + : : +- ^ InputAdapter (69) + : : +- ^ ShuffleQueryStage (68) + : : +- ColumnarExchange (67) + : : +- ^ ProjectExecTransformer (65) + : : +- ^ Scan parquet (64) + : +- ^ InputIteratorTransformer (84) + : +- ^ InputAdapter (83) + : +- ^ ShuffleQueryStage (82) + : +- ColumnarExchange (81) + : +- ^ ProjectExecTransformer (79) + : +- ^ Scan parquet (78) + +- ^ InputIteratorTransformer (98) + +- ^ InputAdapter (97) + +- ^ ShuffleQueryStage (96) + +- ColumnarExchange (95) + +- ^ ProjectExecTransformer (93) + +- ^ Scan parquet (92) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (168) + +- Exchange (167) + +- HashAggregate (166) + +- Exchange (165) + +- HashAggregate (164) + +- Project (163) + +- ShuffledHashJoin Inner BuildRight (162) + :- Exchange (157) + : +- Project (156) + : +- ShuffledHashJoin Inner BuildRight (155) + : :- Exchange (151) + : : +- Project (150) + : : +- ShuffledHashJoin Inner BuildRight (149) + : : :- Exchange (145) + : : : +- Project (144) + : : : +- ShuffledHashJoin Inner BuildRight (143) + : : : :- Exchange (139) + : : : : +- Project (138) + : : : : +- ShuffledHashJoin Inner BuildRight (137) + : : : : :- Exchange (133) + : : : : : +- Project (132) + : : : : : +- ShuffledHashJoin Inner BuildRight (131) + : : : : : :- Exchange (127) + : : : : : : +- Project (126) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (125) + : : : : : : :- Exchange (121) + : : : : : : : +- Project (120) + : : : : : : : +- Filter (119) + : : : : : : : +- Scan parquet (118) + : : : : : : +- Exchange (124) + : : : : : : +- Filter (123) + : : : : : : +- Scan parquet (122) + : : : : : +- Exchange (130) + : : : : : +- Filter (129) + : : : : : +- Scan parquet (128) + : : : : +- Exchange (136) + : : : : +- Filter (135) + : : : : +- Scan parquet (134) + : : : +- Exchange (142) + : : : +- Filter (141) + : : : +- Scan parquet (140) + : : +- Exchange (148) + : : +- Filter (147) + : : +- Scan parquet (146) + : +- Exchange (154) + : +- Filter (153) + : +- Scan parquet (152) + +- Exchange (161) + +- Project (160) + +- Filter (159) + +- Scan parquet (158) (1) Scan parquet @@ -170,744 +162,712 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(22) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(36) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(46) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(47) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(48) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(49) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(50) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(56) ProjectExecTransformer +(51) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(53) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(55) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(60) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(61) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(62) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(63) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(64) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) FilterExecTransformer -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(71) ProjectExecTransformer +(65) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(67) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(69) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(70) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(71) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(72) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(86) ProjectExecTransformer +(79) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(81) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(82) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(83) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(84) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(85) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(86) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(88) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(90) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(92) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) - -(101) ProjectExecTransformer +(93) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(95) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(96) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(97) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(98) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(99) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(100) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(101) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(102) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(103) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(104) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(105) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(106) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(107) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(108) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(109) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(110) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(111) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(112) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(113) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(114) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(115) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(116) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(117) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(118) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(119) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(120) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(121) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(122) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(123) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(124) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(126) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(127) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(128) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(129) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(130) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(132) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(133) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(134) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(135) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(136) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(138) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(139) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(140) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(141) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(142) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(143) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(144) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(145) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(146) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(147) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(148) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(150) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(151) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(152) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(153) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(154) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(156) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(157) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(158) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(159) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(160) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(161) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(162) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(163) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(164) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(165) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(166) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(175) Exchange +(167) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(168) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(169) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt index a811b042b603..8ba93a20d0cd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt @@ -1,126 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (88) + +- ^ SortExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ^ InputAdapter (78) + +- ^ ShuffleQueryStage (77) + +- ColumnarExchange (76) + +- ^ ProjectExecTransformer (74) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ ShuffledHashJoinExecTransformer Inner (71) + :- ^ InputIteratorTransformer (63) + : +- ^ InputAdapter (62) + : +- ^ ShuffleQueryStage (61) + : +- ColumnarExchange (60) + : +- ^ ProjectExecTransformer (58) + : +- ^ ShuffledHashJoinExecTransformer Inner (57) + : :- ^ InputIteratorTransformer (49) + : : +- ^ InputAdapter (48) + : : +- ^ ShuffleQueryStage (47) + : : +- ColumnarExchange (46) + : : +- ^ ProjectExecTransformer (44) + : : +- ^ ShuffledHashJoinExecTransformer Inner (43) + : : :- ^ InputIteratorTransformer (35) + : : : +- ^ InputAdapter (34) + : : : +- ^ ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- ^ ProjectExecTransformer (30) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : : : :- ^ InputIteratorTransformer (21) + : : : : +- ^ InputAdapter (20) + : : : : +- ^ ShuffleQueryStage (19) + : : : : +- ColumnarExchange (18) + : : : : +- ^ ProjectExecTransformer (16) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ ShuffleQueryStage (5) + : : : : : +- ColumnarExchange (4) + : : : : : +- ^ ProjectExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (14) + : : : : +- ^ InputAdapter (13) + : : : : +- ^ ShuffleQueryStage (12) + : : : : +- ColumnarExchange (11) + : : : : +- ^ ProjectExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (28) + : : : +- ^ InputAdapter (27) + : : : +- ^ ShuffleQueryStage (26) + : : : +- ColumnarExchange (25) + : : : +- ^ ProjectExecTransformer (23) + : : : +- ^ Scan parquet (22) + : : +- ^ InputIteratorTransformer (42) + : : +- ^ InputAdapter (41) + : : +- ^ ShuffleQueryStage (40) + : : +- ColumnarExchange (39) + : : +- ^ ProjectExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (56) + : +- ^ InputAdapter (55) + : +- ^ ShuffleQueryStage (54) + : +- ColumnarExchange (53) + : +- ^ ProjectExecTransformer (51) + : +- ^ Scan parquet (50) + +- ^ InputIteratorTransformer (70) + +- ^ InputAdapter (69) + +- ^ ShuffleQueryStage (68) + +- ColumnarExchange (67) + +- ^ ProjectExecTransformer (65) + +- ^ Scan parquet (64) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (126) + +- Exchange (125) + +- HashAggregate (124) + +- Exchange (123) + +- HashAggregate (122) + +- Project (121) + +- ShuffledHashJoin Inner BuildRight (120) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildRight (108) + : : :- Exchange (104) + : : : +- Project (103) + : : : +- ShuffledHashJoin Inner BuildRight (102) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- ShuffledHashJoin Inner BuildLeft (96) + : : : : :- Exchange (92) + : : : : : +- Project (91) + : : : : : +- Filter (90) + : : : : : +- Scan parquet (89) + : : : : +- Exchange (95) + : : : : +- Filter (94) + : : : : +- Scan parquet (93) + : : : +- Exchange (101) + : : : +- Filter (100) + : : : +- Scan parquet (99) + : : +- Exchange (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Exchange (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -130,560 +124,536 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(8) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(22) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(36) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(46) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(47) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(48) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(49) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(50) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: isnotnull(o_orderkey#X) - -(56) ProjectExecTransformer +(51) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(53) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(55) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(60) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(61) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(62) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(63) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(64) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(71) ProjectExecTransformer +(65) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(67) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(69) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(70) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(71) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(72) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(73) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(74) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(76) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(77) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(78) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(79) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(82) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(83) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(84) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(85) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(86) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(88) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(89) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(90) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(91) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(92) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(93) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(94) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(95) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(97) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(98) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(99) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(100) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(101) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(103) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(104) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(105) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(106) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(107) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(109) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(110) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(115) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(116) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(117) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(118) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(119) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(121) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(122) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(123) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(124) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(125) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(126) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt index 0cacd70fdca7..c5d58658acd5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt @@ -1,31 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (27) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (18) + +- ^ SortExecTransformer (16) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (3) + +- ^ ProjectExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (26) + +- Exchange (25) + +- HashAggregate (24) + +- Exchange (23) + +- HashAggregate (22) + +- Project (21) + +- Filter (20) + +- Scan parquet (19) (1) Scan parquet @@ -35,120 +34,116 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) FilterExecTransformer -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Arguments: (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X, CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(3) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) ProjectExecTransformer +(4) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(6) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(6) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(7) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(8) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(9) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(12) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(13) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(14) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(15) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(16) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(18) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(19) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(20) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(21) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(22) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(23) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(24) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(25) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(26) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(27) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt index ae8759a6b86c..af8f35bc4b0e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt @@ -1,85 +1,81 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (83) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (56) + +- TakeOrderedAndProjectExecTransformer (55) + +- ^ ProjectExecTransformer (53) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ ShuffledHashJoinExecTransformer Inner (43) + :- ^ InputIteratorTransformer (35) + : +- ^ InputAdapter (34) + : +- ^ ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- ^ ProjectExecTransformer (30) + : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : :- ^ InputIteratorTransformer (21) + : : +- ^ InputAdapter (20) + : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : +- ColumnarExchange (18) + : : +- ^ ProjectExecTransformer (16) + : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : : +- ColumnarExchange (4) + : : : +- ^ ProjectExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (14) + : : +- ^ InputAdapter (13) + : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : +- ColumnarExchange (11) + : : +- ^ ProjectExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (28) + : +- ^ InputAdapter (27) + : +- ^ ShuffleQueryStage (26), Statistics(X) + : +- ColumnarExchange (25) + : +- ^ ProjectExecTransformer (23) + : +- ^ Scan parquet (22) + +- ^ InputIteratorTransformer (42) + +- ^ InputAdapter (41) + +- ^ ShuffleQueryStage (40), Statistics(X) + +- ColumnarExchange (39) + +- ^ ProjectExecTransformer (37) + +- ^ Scan parquet (36) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (82) + +- HashAggregate (81) + +- Exchange (80) + +- HashAggregate (79) + +- Project (78) + +- ShuffledHashJoin Inner BuildRight (77) + :- Exchange (73) + : +- Project (72) + : +- ShuffledHashJoin Inner BuildRight (71) + : :- Exchange (66) + : : +- Project (65) + : : +- ShuffledHashJoin Inner BuildRight (64) + : : :- Exchange (59) + : : : +- Filter (58) + : : : +- Scan parquet (57) + : : +- Exchange (63) + : : +- Project (62) + : : +- Filter (61) + : : +- Scan parquet (60) + : +- Exchange (70) + : +- Project (69) + : +- Filter (68) + : +- Scan parquet (67) + +- Exchange (76) + +- Filter (75) + +- Scan parquet (74) (1) Scan parquet @@ -89,370 +85,354 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [8]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(22) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(36) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(46) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(48) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(49) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(50) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(51) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(57) ProjectExecTransformer +(53) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(58) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(55) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(57) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(58) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(59) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(60) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(61) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(62) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(63) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(64) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(65) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(66) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(67) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(68) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(69) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(70) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(71) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(72) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(73) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(74) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(75) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(76) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(77) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(78) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(79) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(80) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(81) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(82) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(83) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt index 3b6c477c0cf5..e40294ba7679 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt @@ -1,71 +1,68 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ^ InputAdapter (43) + +- ^ ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- ^ FilterExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ^ InputAdapter (36) + +- ^ ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- ^ ProjectExecTransformer (32) + +- ^ FlushableHashAggregateExecTransformer (31) + +- ^ ProjectExecTransformer (30) + +- ^ ShuffledHashJoinExecTransformer Inner (29) + :- ^ InputIteratorTransformer (21) + : +- ^ InputAdapter (20) + : +- ^ ShuffleQueryStage (19), Statistics(X) + : +- ColumnarExchange (18) + : +- ^ ProjectExecTransformer (16) + : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (14) + : +- ^ InputAdapter (13) + : +- ^ ShuffleQueryStage (12), Statistics(X) + : +- ColumnarExchange (11) + : +- ^ ProjectExecTransformer (9) + : +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (28) + +- ^ InputAdapter (27) + +- ^ ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ Scan parquet (22) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (68) + +- Exchange (67) + +- Filter (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- ShuffledHashJoin Inner BuildRight (61) + :- Exchange (56) + : +- Project (55) + : +- ShuffledHashJoin Inner BuildRight (54) + : :- Exchange (50) + : : +- Filter (49) + : : +- Scan parquet (48) + : +- Exchange (53) + : +- Filter (52) + : +- Scan parquet (51) + +- Exchange (60) + +- Project (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -75,565 +72,548 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(8) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(22) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(31) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(34) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(35) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(36) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(37) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(42) FilterExecTransformer +(39) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(41) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(43) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(45) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(48) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(49) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(50) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(51) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(52) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(53) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(55) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(56) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(57) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(58) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(59) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(60) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(61) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(62) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(63) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(64) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(65) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(69) Filter +(66) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(67) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(68) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 39 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (116) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner (96) - :- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner (85) - : :- ^ InputIteratorTransformer (80) - : : +- ^ InputAdapter (79) - : : +- ^ ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ FilterExecTransformer (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ^ InputAdapter (83) - : +- ^ ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ^ InputAdapter (94) - +- ^ ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (98) + +- ^ ProjectExecTransformer (96) + +- ^ RegularHashAggregateExecTransformer (95) + +- ^ RegularHashAggregateExecTransformer (94) + +- ^ ProjectExecTransformer (93) + +- ^ ShuffledHashJoinExecTransformer Inner (92) + :- ^ InputIteratorTransformer (87) + : +- ^ InputAdapter (86) + : +- ^ ShuffleQueryStage (85), Statistics(X) + : +- ColumnarExchange (84) + : +- ^ ProjectExecTransformer (82) + : +- ^ ShuffledHashJoinExecTransformer Inner (81) + : :- ^ InputIteratorTransformer (76) + : : +- ^ InputAdapter (75) + : : +- ^ ShuffleQueryStage (74), Statistics(X) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ Scan parquet (70) + : +- ^ InputIteratorTransformer (80) + : +- ^ InputAdapter (79) + : +- ^ ShuffleQueryStage (78), Statistics(X) + : +- ReusedExchange (77) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89), Statistics(X) + +- ReusedExchange (88) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (115) + +- HashAggregate (114) + +- Project (113) + +- ShuffledHashJoin Inner BuildRight (112) + :- Exchange (107) + : +- Project (106) + : +- ShuffledHashJoin Inner BuildRight (105) + : :- Exchange (101) + : : +- Filter (100) + : : +- Scan parquet (99) + : +- Exchange (104) + : +- Filter (103) + : +- Scan parquet (102) + +- Exchange (111) + +- Project (110) + +- Filter (109) + +- Scan parquet (108) + + +(70) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) FilterExecTransformer -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(75) ProjectExecTransformer +(71) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(73) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(75) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(77) ReusedExchange [Reuses operator id: 11] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(78) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(79) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(80) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(81) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(86) ProjectExecTransformer +(82) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(84) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(85) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(86) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(87) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(88) ReusedExchange [Reuses operator id: 25] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(89) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(90) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(91) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(97) ProjectExecTransformer +(93) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(94) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(95) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(100) ProjectExecTransformer +(96) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(101) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(98) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(99) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(100) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(101) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(110) Project +(106) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(107) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(108) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(109) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(110) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(111) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(113) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(114) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(115) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(116) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt index 259ddb368b8d..a255925b847c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt @@ -1,50 +1,48 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (47) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ RegularHashAggregateExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ ProjectExecTransformer (18) + +- ^ FlushableHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer Inner (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (46) + +- Exchange (45) + +- HashAggregate (44) + +- Exchange (43) + +- HashAggregate (42) + +- Project (41) + +- ShuffledHashJoin Inner BuildLeft (40) + :- Exchange (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Exchange (39) + +- Project (38) + +- Filter (37) + +- Scan parquet (36) (1) Scan parquet @@ -54,208 +52,200 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: isnotnull(o_orderkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(17) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(24) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(26) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(28) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(30) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(33) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(34) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(35) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(36) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(37) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(38) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(39) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(40) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(41) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(43) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(44) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(45) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(46) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(47) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt index 3f1309f476d9..eb454d4f7f23 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt @@ -1,53 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (51) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer LeftOuter (15) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5), Statistics(X) : +- ColumnarExchange (4) : +- ^ ProjectExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (50) + +- Exchange (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftOuter BuildRight (42) + :- Exchange (37) + : +- Scan parquet (36) + +- Exchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -85,202 +84,198 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) - -(10) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(13) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(16) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(19) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(21) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(25) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(29) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(31) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(33) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(36) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(37) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(39) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(40) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(43) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(44) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(46) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(47) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(48) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(49) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(50) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(51) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt index 68716942cea8..0cd11b2c11f7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt @@ -1,38 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (33) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (21) + +- ^ ProjectExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer Inner (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (32) + +- HashAggregate (31) + +- Project (30) + +- ShuffledHashJoin Inner BuildRight (29) + :- Exchange (25) + : +- Project (24) + : +- Filter (23) + : +- Scan parquet (22) + +- Exchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -42,156 +40,148 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(8) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: isnotnull(p_partkey#X) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(22) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(23) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(24) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(25) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(26) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(27) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(28) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(29) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(30) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(31) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(32) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(33) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt index a9c9e810dd92..96f8900c810c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt @@ -1,45 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner (21) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- AQEShuffleRead (24) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner (19) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FlushableHashAggregateExecTransformer (10) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (39) + +- Exchange (38) + +- Project (37) + +- ShuffledHashJoin Inner BuildLeft (36) + :- Exchange (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Filter (35) + +- HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -49,341 +47,328 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: isnotnull(s_suppkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(10) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(13) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(14) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(15) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(16) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(20) FilterExecTransformer +(18) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(22) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(23) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(24) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(26) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(27) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(28) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(29) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(30) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(31) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(32) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(33) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(37) Filter +(35) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(36) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(39) Project +(37) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(38) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(39) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 18 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (64) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ^ InputAdapter (51) - +- ^ ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ FilterExecTransformer (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ ProjectExecTransformer (51) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- Filter (57) + +- Scan parquet (56) -(43) Scan parquet +(41) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) - -(45) ProjectExecTransformer +(42) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(44) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(46) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(47) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(48) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(49) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) ProjectExecTransformer +(51) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(55) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(56) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(57) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(58) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(59) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(60) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(61) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(65) HashAggregate +(62) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(63) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(64) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt index f4b4bc81f48f..d8bc92e439c3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt @@ -1,64 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (62) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ^ InputAdapter (38) - +- ^ ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ^ InputAdapter (36) + +- ^ ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ FlushableHashAggregateExecTransformer (25) + +- ^ RegularHashAggregateExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ ProjectExecTransformer (18) + +- ^ FlushableHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer Inner (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (61) + +- Exchange (60) + +- HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- ShuffledHashJoin Inner BuildRight (52) + :- Exchange (48) + : +- BroadcastHashJoin LeftAnti BuildRight (47) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Exchange (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -68,282 +66,274 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: isnotnull(ps_partkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Input [2]: [ps_partkey#X, ps_suppkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(17) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(24) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(25) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(26) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(28) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(29) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(30) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(31) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(34) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(38) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(41) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(42) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(44) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(45) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(46) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(48) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(49) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(50) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(51) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(52) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(53) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(54) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(57) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(58) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(59) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(61) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(62) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt index 50c9d211b3d3..a76baac13557 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt @@ -1,59 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ ProjectExecTransformer (29) + +- ^ ShuffledHashJoinExecTransformer Inner (28) + :- ^ ProjectExecTransformer (16) + : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (14) + : +- ^ InputAdapter (13) + : +- ^ ShuffleQueryStage (12), Statistics(X) + : +- ColumnarExchange (11) + : +- ^ ProjectExecTransformer (9) + : +- ^ Scan parquet (8) + +- ^ FilterExecTransformer (27) + +- ^ ProjectExecTransformer (26) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ Scan parquet (17) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (53) + +- HashAggregate (52) + +- Project (51) + +- ShuffledHashJoin Inner BuildRight (50) + :- Project (43) + : +- ShuffledHashJoin Inner BuildRight (42) + : :- Exchange (37) + : : +- Filter (36) + : : +- Scan parquet (35) + : +- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Filter (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -63,262 +60,250 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Arguments: (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [p_partkey#X, p_brand#X, p_container#X] -Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(17) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) FilterExecTransformer -Input [2]: [l_partkey#X, l_quantity#X] -Arguments: isnotnull(l_partkey#X) - -(21) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(19) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(21) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(23) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(27) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(32) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(35) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(36) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(37) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(38) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(39) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(40) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(41) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(43) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(44) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(45) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(46) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(47) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(48) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(49) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(50) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(51) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(52) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(53) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt index a29b40d1e996..36c934dd0416 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt @@ -1,96 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) - :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) - :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (61) + +- TakeOrderedAndProjectExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ ShuffledHashJoinExecTransformer Inner (55) + :- ^ InputIteratorTransformer (39) + : +- ^ InputAdapter (38) + : +- ^ ShuffleQueryStage (37), Statistics(X) + : +- ColumnarExchange (36) + : +- ^ ProjectExecTransformer (34) + : +- ^ ShuffledHashJoinExecTransformer Inner (33) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ ShuffleQueryStage (30), Statistics(X) + : +- ColumnarExchange (29) + : +- ^ ProjectExecTransformer (27) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (26) + : :- ^ InputIteratorTransformer (14) + : : +- ^ InputAdapter (13) + : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : +- ColumnarExchange (11) + : : +- ^ ProjectExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ ProjectExecTransformer (25) + : +- ^ FilterExecTransformer (24) + : +- ^ RegularHashAggregateExecTransformer (23) + : +- ^ InputIteratorTransformer (22) + : +- ^ InputAdapter (21) + : +- ^ ShuffleQueryStage (20), Statistics(X) + : +- ColumnarExchange (19) + : +- ^ ProjectExecTransformer (17) + : +- ^ FlushableHashAggregateExecTransformer (16) + : +- ^ Scan parquet (15) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (54) + :- ^ InputIteratorTransformer (46) + : +- ^ InputAdapter (45) + : +- ^ ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- ^ ProjectExecTransformer (41) + : +- ^ Scan parquet (40) + +- ^ ProjectExecTransformer (53) + +- ^ FilterExecTransformer (52) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ^ InputAdapter (49) + +- ^ ShuffleQueryStage (48), Statistics(X) + +- ReusedExchange (47) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- HashAggregate (91) + +- Project (90) + +- ShuffledHashJoin Inner BuildRight (89) + :- Exchange (78) + : +- Project (77) + : +- ShuffledHashJoin Inner BuildLeft (76) + : :- Exchange (64) + : : +- Filter (63) + : : +- Scan parquet (62) + : +- Exchange (75) + : +- ShuffledHashJoin LeftSemi BuildRight (74) + : :- Exchange (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Project (73) + : +- Filter (72) + : +- HashAggregate (71) + : +- Exchange (70) + : +- HashAggregate (69) + : +- Scan parquet (68) + +- ShuffledHashJoin LeftSemi BuildRight (88) + :- Exchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- Project (87) + +- Filter (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Scan parquet (82) (1) Scan parquet @@ -100,432 +97,420 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_name#X] -Arguments: isnotnull(c_custkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X] Input [2]: [c_custkey#X, c_name#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(15) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(16) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(17) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(19) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(20) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(21) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(22) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(24) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(25) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(26) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(27) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(29) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(30) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(31) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(32) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(34) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(36) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(37) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(38) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(39) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(40) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) FilterExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: isnotnull(l_orderkey#X) - -(44) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(45) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(47) ReusedExchange [Reuses operator id: 19] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(48) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(49) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(50) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(52) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(54) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(56) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(60) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(62) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(63) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(64) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(65) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(66) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(67) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(68) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(69) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(70) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(71) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(72) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(73) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(76) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(77) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(78) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(79) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(80) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(81) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(82) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(83) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(84) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(85) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(86) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(87) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(89) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(90) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(91) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(92) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt index 69b8adaee61d..abcddb7ca47a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt @@ -1,37 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (32) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer Inner (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (31) + +- HashAggregate (30) + +- Project (29) + +- ShuffledHashJoin Inner BuildRight (28) + :- Exchange (24) + : +- Project (23) + : +- Filter (22) + : +- Scan parquet (21) + +- Exchange (27) + +- Filter (26) + +- Scan parquet (25) (1) Scan parquet @@ -41,152 +39,144 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) FilterExecTransformer -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Arguments: ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(21) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(22) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(23) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(24) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(25) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(26) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(27) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(28) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(29) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(30) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(31) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(32) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt index 4fa43644b568..e24568ef1f6c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt @@ -1,119 +1,114 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (118) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) - : :- ^ InputIteratorTransformer (31) - : : +- ^ InputAdapter (30) - : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) - : :- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ^ InputAdapter (42) - : +- ^ ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (78) + +- AQEShuffleRead (77) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ ShuffledHashJoinExecTransformer Inner (72) + :- ^ InputIteratorTransformer (64) + : +- ^ InputAdapter (63) + : +- ^ ShuffleQueryStage (62), Statistics(X) + : +- ColumnarExchange (61) + : +- ^ ProjectExecTransformer (59) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (58) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (57) + : +- ^ InputAdapter (56) + : +- ^ ShuffleQueryStage (55), Statistics(X) + : +- ColumnarExchange (54) + : +- ^ ProjectExecTransformer (52) + : +- ^ ShuffledHashJoinExecTransformer Inner (51) + : :- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ ShuffleQueryStage (26), Statistics(X) + : : +- ColumnarExchange (25) + : : +- ^ ProjectExecTransformer (23) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (22) + : : :- ^ InputIteratorTransformer (14) + : : : +- ^ InputAdapter (13) + : : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : : +- ColumnarExchange (11) + : : : +- ^ ProjectExecTransformer (9) + : : : +- ^ Scan parquet (8) + : : +- ^ InputIteratorTransformer (21) + : : +- ^ InputAdapter (20) + : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : +- ColumnarExchange (18) + : : +- ^ ProjectExecTransformer (16) + : : +- ^ Scan parquet (15) + : +- ^ InputIteratorTransformer (50) + : +- ^ InputAdapter (49) + : +- ^ ShuffleQueryStage (48), Statistics(X) + : +- ColumnarExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ FilterExecTransformer (44) + : +- ^ ProjectExecTransformer (43) + : +- ^ RegularHashAggregateExecTransformer (42) + : +- ^ RegularHashAggregateExecTransformer (41) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (40) + : :- ^ InputIteratorTransformer (35) + : : +- ^ InputAdapter (34) + : : +- ^ ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ Scan parquet (29) + : +- ^ InputIteratorTransformer (39) + : +- ^ InputAdapter (38) + : +- ^ ShuffleQueryStage (37), Statistics(X) + : +- ReusedExchange (36) + +- ^ InputIteratorTransformer (71) + +- ^ InputAdapter (70) + +- ^ ShuffleQueryStage (69), Statistics(X) + +- ColumnarExchange (68) + +- ^ ProjectExecTransformer (66) + +- ^ Scan parquet (65) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (117) + +- Exchange (116) + +- Project (115) + +- ShuffledHashJoin Inner BuildRight (114) + :- Exchange (109) + : +- Project (108) + : +- ShuffledHashJoin LeftSemi BuildRight (107) + : :- Exchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- Exchange (106) + : +- Project (105) + : +- ShuffledHashJoin Inner BuildLeft (104) + : :- Exchange (90) + : : +- ShuffledHashJoin LeftSemi BuildRight (89) + : : :- Exchange (84) + : : : +- Filter (83) + : : : +- Scan parquet (82) + : : +- Exchange (88) + : : +- Project (87) + : : +- Filter (86) + : : +- Scan parquet (85) + : +- Exchange (103) + : +- Filter (102) + : +- HashAggregate (101) + : +- HashAggregate (100) + : +- ShuffledHashJoin LeftSemi BuildRight (99) + : :- Exchange (94) + : : +- Project (93) + : : +- Filter (92) + : : +- Scan parquet (91) + : +- Exchange (98) + : +- Project (97) + : +- Filter (96) + : +- Scan parquet (95) + +- Exchange (113) + +- Project (112) + +- Filter (111) + +- Scan parquet (110) (1) Scan parquet @@ -123,520 +118,500 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: isnotnull(s_nationkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(15) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) - -(19) ProjectExecTransformer +(16) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(18) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(19) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(20) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(21) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(29) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) FilterExecTransformer -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) - -(34) ProjectExecTransformer +(30) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(34) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(36) ReusedExchange [Reuses operator id: 18] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(37) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(38) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(39) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(40) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(42) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(43) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(44) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(47) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(48) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(49) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(50) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(51) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(52) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(54) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(55) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(56) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(57) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(58) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(59) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(61) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(62) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(63) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(64) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) - -(71) ProjectExecTransformer +(66) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(68) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(69) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(70) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(71) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(72) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(73) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(75) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(77) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(78) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(79) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(80) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(81) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(82) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(83) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(84) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(86) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(88) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(89) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(95) Exchange +(90) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(91) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(92) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(93) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(94) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(95) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(96) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(97) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(98) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(99) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(105) HashAggregate +(100) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(101) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(102) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(103) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(105) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(106) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(113) Project +(108) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(109) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(111) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(112) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(113) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(120) Project +(115) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(116) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(117) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(118) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt index f17e6fa164a8..fec143f3c82c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt @@ -1,113 +1,108 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (113) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner (70) - :- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) - : :- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ^ InputAdapter (22) - : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ^ InputAdapter (31) - : : +- ^ ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ^ InputAdapter (68) - +- ^ ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (76) + +- ^ RegularHashAggregateExecTransformer (74) + +- ^ InputIteratorTransformer (73) + +- ^ InputAdapter (72) + +- ^ ShuffleQueryStage (71), Statistics(X) + +- ColumnarExchange (70) + +- ^ ProjectExecTransformer (68) + +- ^ FlushableHashAggregateExecTransformer (67) + +- ^ ProjectExecTransformer (66) + +- ^ ShuffledHashJoinExecTransformer Inner (65) + :- ^ InputIteratorTransformer (57) + : +- ^ InputAdapter (56) + : +- ^ ShuffleQueryStage (55), Statistics(X) + : +- ColumnarExchange (54) + : +- ^ ProjectExecTransformer (52) + : +- ^ ShuffledHashJoinExecTransformer Inner (51) + : :- ^ InputIteratorTransformer (43) + : : +- ^ InputAdapter (42) + : : +- ^ ShuffleQueryStage (41), Statistics(X) + : : +- ColumnarExchange (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer Inner (37) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : : +- ColumnarExchange (4) + : : : +- ^ ProjectExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (36) + : : +- ^ InputAdapter (35) + : : +- ^ ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- ^ ProjectExecTransformer (31) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (30) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (22) + : : : :- ^ InputIteratorTransformer (14) + : : : : +- ^ InputAdapter (13) + : : : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : : : +- ColumnarExchange (11) + : : : : +- ^ ProjectExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (21) + : : : +- ^ InputAdapter (20) + : : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : : +- ColumnarExchange (18) + : : : +- ^ ProjectExecTransformer (16) + : : : +- ^ Scan parquet (15) + : : +- ^ InputIteratorTransformer (29) + : : +- ^ InputAdapter (28) + : : +- ^ ShuffleQueryStage (27), Statistics(X) + : : +- ColumnarExchange (26) + : : +- ^ ProjectExecTransformer (24) + : : +- ^ Scan parquet (23) + : +- ^ InputIteratorTransformer (50) + : +- ^ InputAdapter (49) + : +- ^ ShuffleQueryStage (48), Statistics(X) + : +- ColumnarExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ Scan parquet (44) + +- ^ InputIteratorTransformer (64) + +- ^ InputAdapter (63) + +- ^ ShuffleQueryStage (62), Statistics(X) + +- ColumnarExchange (61) + +- ^ ProjectExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (112) + +- HashAggregate (111) + +- Exchange (110) + +- HashAggregate (109) + +- Project (108) + +- ShuffledHashJoin Inner BuildRight (107) + :- Exchange (102) + : +- Project (101) + : +- ShuffledHashJoin Inner BuildRight (100) + : :- Exchange (95) + : : +- Project (94) + : : +- ShuffledHashJoin Inner BuildLeft (93) + : : :- Exchange (79) + : : : +- Filter (78) + : : : +- Scan parquet (77) + : : +- Exchange (92) + : : +- ShuffledHashJoin LeftAnti BuildRight (91) + : : :- ShuffledHashJoin LeftSemi BuildRight (86) + : : : :- Exchange (83) + : : : : +- Project (82) + : : : : +- Filter (81) + : : : : +- Scan parquet (80) + : : : +- Exchange (85) + : : : +- Scan parquet (84) + : : +- Exchange (90) + : : +- Project (89) + : : +- Filter (88) + : : +- Scan parquet (87) + : +- Exchange (99) + : +- Project (98) + : +- Filter (97) + : +- Scan parquet (96) + +- Exchange (106) + +- Project (105) + +- Filter (104) + +- Scan parquet (103) (1) Scan parquet @@ -117,502 +112,482 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(15) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(23) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) - -(27) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(28) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(31) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(40) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(41) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(42) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(43) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(44) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderstatus#X] -Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) - -(49) ProjectExecTransformer +(45) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(47) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(48) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(49) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(50) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(51) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(54) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(55) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(56) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(57) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(58) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) - -(64) ProjectExecTransformer +(59) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(61) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(62) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(63) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(64) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(65) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(66) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(67) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(69) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(70) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(71) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(72) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(73) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(74) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(76) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(77) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(78) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(79) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(80) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(81) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(82) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(83) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(84) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(85) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(86) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(87) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(88) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(89) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(91) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(92) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(93) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(99) Project +(94) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(95) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(96) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(97) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(98) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(99) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(106) Project +(101) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(102) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(103) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(104) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(105) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(106) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(108) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(109) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(110) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(111) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(112) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(113) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt index f8ce5b6c7485..2f5f9d303931 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt @@ -1,47 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ^ InputAdapter (29) - +- ^ ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ RegularHashAggregateExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ ProjectExecTransformer (18) + +- ^ FlushableHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer LeftAnti (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- ShuffledHashJoin LeftAnti BuildRight (38) + :- Exchange (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Exchange (37) + +- Scan parquet (36) (1) Scan parquet @@ -51,330 +50,300 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(8) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(9) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(11) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(12) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(13) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(14) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(16) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(17) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(22) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(24) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(26) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(28) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(30) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(33) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(34) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(35) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(36) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(37) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(38) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(39) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(40) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(41) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(42) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(43) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(44) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (56) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51), Statistics(X) + +- ColumnarExchange (50) + +- ^ FlushableHashAggregateExecTransformer (48) + +- ^ ProjectExecTransformer (47) + +- ^ Scan parquet (46) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (62) + +- Exchange (61) + +- HashAggregate (60) + +- Project (59) + +- Filter (58) + +- Scan parquet (57) -(47) Scan parquet +(46) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) FilterExecTransformer -Input [2]: [c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) - -(49) ProjectExecTransformer +(47) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(48) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(50) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(52) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(57) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(58) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(59) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(60) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(61) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(62) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] -Arguments: isFinalPlan=true - -Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) -+- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) -+- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) \ No newline at end of file +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt index 8492e97cdbcc..48b0e5f28558 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt @@ -1,60 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (36) + +- TakeOrderedAndProjectExecTransformer (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ ProjectExecTransformer (30) + +- ^ ShuffledHashJoinExecTransformer Inner (29) + :- ^ InputIteratorTransformer (21) + : +- ^ InputAdapter (20) + : +- ^ ShuffleQueryStage (19), Statistics(X) + : +- ColumnarExchange (18) + : +- ^ ProjectExecTransformer (16) + : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (14) + : +- ^ InputAdapter (13) + : +- ^ ShuffleQueryStage (12), Statistics(X) + : +- ColumnarExchange (11) + : +- ^ ProjectExecTransformer (9) + : +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (28) + +- ^ InputAdapter (27) + +- ^ ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ Scan parquet (22) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Project (52) + +- ShuffledHashJoin Inner BuildRight (51) + :- Exchange (46) + : +- Project (45) + : +- ShuffledHashJoin Inner BuildLeft (44) + : :- Exchange (40) + : : +- Project (39) + : : +- Filter (38) + : : +- Scan parquet (37) + : +- Exchange (43) + : +- Filter (42) + : +- Scan parquet (41) + +- Exchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -64,256 +61,244 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_mktsegment#X] -Arguments: ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(22) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(36) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(37) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(35) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(37) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(38) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(39) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(40) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(41) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(42) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(43) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(44) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(45) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(46) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(47) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(48) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(49) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(50) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(52) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(53) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(54) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(55) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt index 22e9480fffc8..33b40f964185 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt @@ -1,51 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (48) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ RegularHashAggregateExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ ProjectExecTransformer (18) + +- ^ FlushableHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (47) + +- Exchange (46) + +- HashAggregate (45) + +- Exchange (44) + +- HashAggregate (43) + +- Project (42) + +- ShuffledHashJoin LeftSemi BuildRight (41) + :- Exchange (36) + : +- Project (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Exchange (40) + +- Project (39) + +- Filter (38) + +- Scan parquet (37) (1) Scan parquet @@ -55,212 +53,204 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Arguments: ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(17) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(24) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(26) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(28) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(30) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(33) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(34) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(35) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(36) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(37) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(38) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(39) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(40) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(42) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(43) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(45) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(46) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(47) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(48) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt index 59273229a1ae..719d3611db67 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt @@ -1,127 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (88) + +- ^ SortExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ^ InputAdapter (78) + +- ^ ShuffleQueryStage (77), Statistics(X) + +- ColumnarExchange (76) + +- ^ ProjectExecTransformer (74) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ ShuffledHashJoinExecTransformer Inner (71) + :- ^ InputIteratorTransformer (63) + : +- ^ InputAdapter (62) + : +- ^ ShuffleQueryStage (61), Statistics(X) + : +- ColumnarExchange (60) + : +- ^ ProjectExecTransformer (58) + : +- ^ ShuffledHashJoinExecTransformer Inner (57) + : :- ^ InputIteratorTransformer (49) + : : +- ^ InputAdapter (48) + : : +- ^ ShuffleQueryStage (47), Statistics(X) + : : +- ColumnarExchange (46) + : : +- ^ ProjectExecTransformer (44) + : : +- ^ ShuffledHashJoinExecTransformer Inner (43) + : : :- ^ InputIteratorTransformer (35) + : : : +- ^ InputAdapter (34) + : : : +- ^ ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- ^ ProjectExecTransformer (30) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : : : :- ^ InputIteratorTransformer (21) + : : : : +- ^ InputAdapter (20) + : : : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : : : +- ColumnarExchange (18) + : : : : +- ^ ProjectExecTransformer (16) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : : : : +- ColumnarExchange (4) + : : : : : +- ^ ProjectExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (14) + : : : : +- ^ InputAdapter (13) + : : : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : : : +- ColumnarExchange (11) + : : : : +- ^ ProjectExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (28) + : : : +- ^ InputAdapter (27) + : : : +- ^ ShuffleQueryStage (26), Statistics(X) + : : : +- ColumnarExchange (25) + : : : +- ^ ProjectExecTransformer (23) + : : : +- ^ Scan parquet (22) + : : +- ^ InputIteratorTransformer (42) + : : +- ^ InputAdapter (41) + : : +- ^ ShuffleQueryStage (40), Statistics(X) + : : +- ColumnarExchange (39) + : : +- ^ ProjectExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (56) + : +- ^ InputAdapter (55) + : +- ^ ShuffleQueryStage (54), Statistics(X) + : +- ColumnarExchange (53) + : +- ^ ProjectExecTransformer (51) + : +- ^ Scan parquet (50) + +- ^ InputIteratorTransformer (70) + +- ^ InputAdapter (69) + +- ^ ShuffleQueryStage (68), Statistics(X) + +- ColumnarExchange (67) + +- ^ ProjectExecTransformer (65) + +- ^ Scan parquet (64) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (127) + +- Exchange (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildRight (108) + : : :- Exchange (104) + : : : +- Project (103) + : : : +- ShuffledHashJoin Inner BuildRight (102) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- ShuffledHashJoin Inner BuildLeft (96) + : : : : :- Exchange (91) + : : : : : +- Filter (90) + : : : : : +- Scan parquet (89) + : : : : +- Exchange (95) + : : : : +- Project (94) + : : : : +- Filter (93) + : : : : +- Scan parquet (92) + : : : +- Exchange (101) + : : : +- Filter (100) + : : : +- Scan parquet (99) + : : +- Exchange (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Exchange (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -131,564 +125,540 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(22) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(36) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(46) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(47) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(48) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(49) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(50) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) FilterExecTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(56) ProjectExecTransformer +(51) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(53) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(55) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(60) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(61) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(62) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(63) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(64) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) - -(71) ProjectExecTransformer +(65) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(67) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(68) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(69) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(70) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(71) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(72) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(73) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(74) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(76) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(77) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(78) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(79) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(82) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(83) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(84) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(85) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(86) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(88) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(89) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(90) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(91) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(92) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(93) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(94) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(95) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(97) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(98) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(99) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(100) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(101) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(103) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(104) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(105) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(106) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(107) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(109) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(110) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(111) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(112) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(113) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(115) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(116) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(117) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(118) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(119) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(120) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(122) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(123) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(124) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(125) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(132) Exchange +(126) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(127) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt index 15f764040184..2e51f22a6b0e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt @@ -1,23 +1,22 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (18) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (11) + +- ^ RegularHashAggregateExecTransformer (9) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ ShuffleQueryStage (6), Statistics(X) + +- ColumnarExchange (5) + +- ^ FlushableHashAggregateExecTransformer (3) + +- ^ ProjectExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (17) + +- Exchange (16) + +- HashAggregate (15) + +- Project (14) + +- Filter (13) + +- Scan parquet (12) (1) Scan parquet @@ -27,86 +26,82 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(3) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(5) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(5) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(7) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(9) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(11) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(12) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(13) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(14) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(15) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(16) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(17) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(18) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt index 2aa77120c693..eca3692340aa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt @@ -1,122 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (123) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ^ InputAdapter (86) - +- ^ ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner (73) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ^ InputAdapter (71) - +- ^ ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ^ InputAdapter (81) + +- ^ ShuffleQueryStage (80), Statistics(X) + +- ColumnarExchange (79) + +- ^ RegularHashAggregateExecTransformer (77) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ FlushableHashAggregateExecTransformer (70) + +- ^ ProjectExecTransformer (69) + +- ^ ShuffledHashJoinExecTransformer Inner (68) + :- ^ InputIteratorTransformer (63) + : +- ^ InputAdapter (62) + : +- ^ ShuffleQueryStage (61), Statistics(X) + : +- ColumnarExchange (60) + : +- ^ ProjectExecTransformer (58) + : +- ^ ShuffledHashJoinExecTransformer Inner (57) + : :- ^ InputIteratorTransformer (49) + : : +- ^ InputAdapter (48) + : : +- ^ ShuffleQueryStage (47), Statistics(X) + : : +- ColumnarExchange (46) + : : +- ^ ProjectExecTransformer (44) + : : +- ^ ShuffledHashJoinExecTransformer Inner (43) + : : :- ^ InputIteratorTransformer (35) + : : : +- ^ InputAdapter (34) + : : : +- ^ ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- ^ ProjectExecTransformer (30) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : : : :- ^ InputIteratorTransformer (21) + : : : : +- ^ InputAdapter (20) + : : : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : : : +- ColumnarExchange (18) + : : : : +- ^ ProjectExecTransformer (16) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : : : : +- ColumnarExchange (4) + : : : : : +- ^ ProjectExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (14) + : : : : +- ^ InputAdapter (13) + : : : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : : : +- ColumnarExchange (11) + : : : : +- ^ ProjectExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (28) + : : : +- ^ InputAdapter (27) + : : : +- ^ ShuffleQueryStage (26), Statistics(X) + : : : +- ColumnarExchange (25) + : : : +- ^ ProjectExecTransformer (23) + : : : +- ^ Scan parquet (22) + : : +- ^ InputIteratorTransformer (42) + : : +- ^ InputAdapter (41) + : : +- ^ ShuffleQueryStage (40), Statistics(X) + : : +- ColumnarExchange (39) + : : +- ^ ProjectExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (56) + : +- ^ InputAdapter (55) + : +- ^ ShuffleQueryStage (54), Statistics(X) + : +- ColumnarExchange (53) + : +- ^ ProjectExecTransformer (51) + : +- ^ Scan parquet (50) + +- ^ InputIteratorTransformer (67) + +- ^ InputAdapter (66) + +- ^ ShuffleQueryStage (65), Statistics(X) + +- ReusedExchange (64) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (122) + +- Exchange (121) + +- HashAggregate (120) + +- Exchange (119) + +- HashAggregate (118) + +- Project (117) + +- ShuffledHashJoin Inner BuildRight (116) + :- Exchange (112) + : +- Project (111) + : +- ShuffledHashJoin Inner BuildRight (110) + : :- Exchange (106) + : : +- Project (105) + : : +- ShuffledHashJoin Inner BuildRight (104) + : : :- Exchange (100) + : : : +- Project (99) + : : : +- ShuffledHashJoin Inner BuildRight (98) + : : : :- Exchange (94) + : : : : +- Project (93) + : : : : +- ShuffledHashJoin Inner BuildLeft (92) + : : : : :- Exchange (88) + : : : : : +- Filter (87) + : : : : : +- Scan parquet (86) + : : : : +- Exchange (91) + : : : : +- Filter (90) + : : : : +- Scan parquet (89) + : : : +- Exchange (97) + : : : +- Filter (96) + : : : +- Scan parquet (95) + : : +- Exchange (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Exchange (109) + : +- Filter (108) + : +- Scan parquet (107) + +- Exchange (115) + +- Filter (114) + +- Scan parquet (113) (1) Scan parquet @@ -126,536 +121,516 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(22) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(46) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(47) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(48) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(49) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(50) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) - -(56) ProjectExecTransformer +(51) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(53) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(55) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(60) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(61) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(62) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(63) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(64) ReusedExchange [Reuses operator id: 53] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(65) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(66) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(67) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(68) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(69) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(70) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(73) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(74) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(75) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(76) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(77) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(79) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(80) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(81) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(82) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(83) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(86) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(87) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(88) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(89) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(90) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(91) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(93) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(94) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(95) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(96) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(97) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(99) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(100) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(101) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(102) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(103) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(105) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(106) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(107) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(108) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(109) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(110) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(111) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(112) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(113) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(114) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(115) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(117) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(118) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(119) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(120) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(121) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(122) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(123) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt index ab8ef221e503..1883d0df585f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt @@ -1,166 +1,158 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (169) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ^ InputAdapter (121) - +- ^ ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ^ InputAdapter (114) - +- ^ ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner (107) - :- ^ InputIteratorTransformer (98) - : +- ^ InputAdapter (97) - : +- ^ ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner (92) - : :- ^ InputIteratorTransformer (83) - : : +- ^ InputAdapter (82) - : : +- ^ ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ^ InputAdapter (67) - : : : +- ^ ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ^ InputAdapter (52) - : : : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ^ InputAdapter (37) - : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ^ InputAdapter (22) - : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ^ InputAdapter (15) - : : : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ^ InputAdapter (30) - : : : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ^ InputAdapter (45) - : : : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ^ InputAdapter (60) - : : : +- ^ ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ^ InputAdapter (105) - +- ^ ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (117) + +- ^ SortExecTransformer (115) + +- ^ InputIteratorTransformer (114) + +- ^ InputAdapter (113) + +- ^ ShuffleQueryStage (112), Statistics(X) + +- ColumnarExchange (111) + +- ^ ProjectExecTransformer (109) + +- ^ RegularHashAggregateExecTransformer (108) + +- ^ InputIteratorTransformer (107) + +- ^ InputAdapter (106) + +- ^ ShuffleQueryStage (105), Statistics(X) + +- ColumnarExchange (104) + +- ^ ProjectExecTransformer (102) + +- ^ FlushableHashAggregateExecTransformer (101) + +- ^ ProjectExecTransformer (100) + +- ^ ShuffledHashJoinExecTransformer Inner (99) + :- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ ShuffledHashJoinExecTransformer Inner (85) + : :- ^ InputIteratorTransformer (77) + : : +- ^ InputAdapter (76) + : : +- ^ ShuffleQueryStage (75), Statistics(X) + : : +- ColumnarExchange (74) + : : +- ^ ProjectExecTransformer (72) + : : +- ^ ShuffledHashJoinExecTransformer Inner (71) + : : :- ^ InputIteratorTransformer (63) + : : : +- ^ InputAdapter (62) + : : : +- ^ ShuffleQueryStage (61), Statistics(X) + : : : +- ColumnarExchange (60) + : : : +- ^ ProjectExecTransformer (58) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (57) + : : : :- ^ InputIteratorTransformer (49) + : : : : +- ^ InputAdapter (48) + : : : : +- ^ ShuffleQueryStage (47), Statistics(X) + : : : : +- ColumnarExchange (46) + : : : : +- ^ ProjectExecTransformer (44) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (43) + : : : : :- ^ InputIteratorTransformer (35) + : : : : : +- ^ InputAdapter (34) + : : : : : +- ^ ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- ^ ProjectExecTransformer (30) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : : : : : :- ^ InputIteratorTransformer (21) + : : : : : : +- ^ InputAdapter (20) + : : : : : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : : : : : +- ColumnarExchange (18) + : : : : : : +- ^ ProjectExecTransformer (16) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : : : : : :- ^ InputIteratorTransformer (7) + : : : : : : : +- ^ InputAdapter (6) + : : : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : : : : : : +- ColumnarExchange (4) + : : : : : : : +- ^ ProjectExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (14) + : : : : : : +- ^ InputAdapter (13) + : : : : : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : : : : : +- ColumnarExchange (11) + : : : : : : +- ^ ProjectExecTransformer (9) + : : : : : : +- ^ Scan parquet (8) + : : : : : +- ^ InputIteratorTransformer (28) + : : : : : +- ^ InputAdapter (27) + : : : : : +- ^ ShuffleQueryStage (26), Statistics(X) + : : : : : +- ColumnarExchange (25) + : : : : : +- ^ ProjectExecTransformer (23) + : : : : : +- ^ Scan parquet (22) + : : : : +- ^ InputIteratorTransformer (42) + : : : : +- ^ InputAdapter (41) + : : : : +- ^ ShuffleQueryStage (40), Statistics(X) + : : : : +- ColumnarExchange (39) + : : : : +- ^ ProjectExecTransformer (37) + : : : : +- ^ Scan parquet (36) + : : : +- ^ InputIteratorTransformer (56) + : : : +- ^ InputAdapter (55) + : : : +- ^ ShuffleQueryStage (54), Statistics(X) + : : : +- ColumnarExchange (53) + : : : +- ^ ProjectExecTransformer (51) + : : : +- ^ Scan parquet (50) + : : +- ^ InputIteratorTransformer (70) + : : +- ^ InputAdapter (69) + : : +- ^ ShuffleQueryStage (68), Statistics(X) + : : +- ColumnarExchange (67) + : : +- ^ ProjectExecTransformer (65) + : : +- ^ Scan parquet (64) + : +- ^ InputIteratorTransformer (84) + : +- ^ InputAdapter (83) + : +- ^ ShuffleQueryStage (82), Statistics(X) + : +- ColumnarExchange (81) + : +- ^ ProjectExecTransformer (79) + : +- ^ Scan parquet (78) + +- ^ InputIteratorTransformer (98) + +- ^ InputAdapter (97) + +- ^ ShuffleQueryStage (96), Statistics(X) + +- ColumnarExchange (95) + +- ^ ProjectExecTransformer (93) + +- ^ Scan parquet (92) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (168) + +- Exchange (167) + +- HashAggregate (166) + +- Exchange (165) + +- HashAggregate (164) + +- Project (163) + +- ShuffledHashJoin Inner BuildRight (162) + :- Exchange (157) + : +- Project (156) + : +- ShuffledHashJoin Inner BuildRight (155) + : :- Exchange (151) + : : +- Project (150) + : : +- ShuffledHashJoin Inner BuildRight (149) + : : :- Exchange (145) + : : : +- Project (144) + : : : +- ShuffledHashJoin Inner BuildRight (143) + : : : :- Exchange (139) + : : : : +- Project (138) + : : : : +- ShuffledHashJoin Inner BuildRight (137) + : : : : :- Exchange (133) + : : : : : +- Project (132) + : : : : : +- ShuffledHashJoin Inner BuildRight (131) + : : : : : :- Exchange (127) + : : : : : : +- Project (126) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (125) + : : : : : : :- Exchange (121) + : : : : : : : +- Project (120) + : : : : : : : +- Filter (119) + : : : : : : : +- Scan parquet (118) + : : : : : : +- Exchange (124) + : : : : : : +- Filter (123) + : : : : : : +- Scan parquet (122) + : : : : : +- Exchange (130) + : : : : : +- Filter (129) + : : : : : +- Scan parquet (128) + : : : : +- Exchange (136) + : : : : +- Filter (135) + : : : : +- Scan parquet (134) + : : : +- Exchange (142) + : : : +- Filter (141) + : : : +- Scan parquet (140) + : : +- Exchange (148) + : : +- Filter (147) + : : +- Scan parquet (146) + : +- Exchange (154) + : +- Filter (153) + : +- Scan parquet (152) + +- Exchange (161) + +- Project (160) + +- Filter (159) + +- Scan parquet (158) (1) Scan parquet @@ -170,744 +162,712 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(22) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(36) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(46) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(47) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(48) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(49) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(50) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(56) ProjectExecTransformer +(51) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(53) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(55) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(60) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(61) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(62) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(63) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(64) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) FilterExecTransformer -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(71) ProjectExecTransformer +(65) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(67) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(69) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(70) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(71) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(72) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(86) ProjectExecTransformer +(79) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(81) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(82) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(83) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(84) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(85) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(86) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(88) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(90) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(92) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) - -(101) ProjectExecTransformer +(93) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(95) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(96) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(97) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(98) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(99) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(100) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(101) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(102) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(103) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(104) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(105) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(106) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(107) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(108) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(109) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(110) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(111) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(112) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(113) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(114) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(115) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(116) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(117) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(118) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(119) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(120) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(121) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(122) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(123) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(124) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(126) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(127) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(128) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(129) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(130) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(132) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(133) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(134) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(135) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(136) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(138) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(139) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(140) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(141) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(142) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(143) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(144) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(145) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(146) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(147) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(148) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(150) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(151) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(152) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(153) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(154) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(156) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(157) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(158) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(159) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(160) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(161) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(162) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(163) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(164) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(165) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(166) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(175) Exchange +(167) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(168) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(169) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt index 08e6c30a8da3..feffe6710113 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt @@ -1,126 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (88) + +- ^ SortExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ^ InputAdapter (78) + +- ^ ShuffleQueryStage (77), Statistics(X) + +- ColumnarExchange (76) + +- ^ ProjectExecTransformer (74) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ ShuffledHashJoinExecTransformer Inner (71) + :- ^ InputIteratorTransformer (63) + : +- ^ InputAdapter (62) + : +- ^ ShuffleQueryStage (61), Statistics(X) + : +- ColumnarExchange (60) + : +- ^ ProjectExecTransformer (58) + : +- ^ ShuffledHashJoinExecTransformer Inner (57) + : :- ^ InputIteratorTransformer (49) + : : +- ^ InputAdapter (48) + : : +- ^ ShuffleQueryStage (47), Statistics(X) + : : +- ColumnarExchange (46) + : : +- ^ ProjectExecTransformer (44) + : : +- ^ ShuffledHashJoinExecTransformer Inner (43) + : : :- ^ InputIteratorTransformer (35) + : : : +- ^ InputAdapter (34) + : : : +- ^ ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- ^ ProjectExecTransformer (30) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : : : :- ^ InputIteratorTransformer (21) + : : : : +- ^ InputAdapter (20) + : : : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : : : +- ColumnarExchange (18) + : : : : +- ^ ProjectExecTransformer (16) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : : : : +- ColumnarExchange (4) + : : : : : +- ^ ProjectExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (14) + : : : : +- ^ InputAdapter (13) + : : : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : : : +- ColumnarExchange (11) + : : : : +- ^ ProjectExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (28) + : : : +- ^ InputAdapter (27) + : : : +- ^ ShuffleQueryStage (26), Statistics(X) + : : : +- ColumnarExchange (25) + : : : +- ^ ProjectExecTransformer (23) + : : : +- ^ Scan parquet (22) + : : +- ^ InputIteratorTransformer (42) + : : +- ^ InputAdapter (41) + : : +- ^ ShuffleQueryStage (40), Statistics(X) + : : +- ColumnarExchange (39) + : : +- ^ ProjectExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (56) + : +- ^ InputAdapter (55) + : +- ^ ShuffleQueryStage (54), Statistics(X) + : +- ColumnarExchange (53) + : +- ^ ProjectExecTransformer (51) + : +- ^ Scan parquet (50) + +- ^ InputIteratorTransformer (70) + +- ^ InputAdapter (69) + +- ^ ShuffleQueryStage (68), Statistics(X) + +- ColumnarExchange (67) + +- ^ ProjectExecTransformer (65) + +- ^ Scan parquet (64) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (126) + +- Exchange (125) + +- HashAggregate (124) + +- Exchange (123) + +- HashAggregate (122) + +- Project (121) + +- ShuffledHashJoin Inner BuildRight (120) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildRight (108) + : : :- Exchange (104) + : : : +- Project (103) + : : : +- ShuffledHashJoin Inner BuildRight (102) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- ShuffledHashJoin Inner BuildLeft (96) + : : : : :- Exchange (92) + : : : : : +- Project (91) + : : : : : +- Filter (90) + : : : : : +- Scan parquet (89) + : : : : +- Exchange (95) + : : : : +- Filter (94) + : : : : +- Scan parquet (93) + : : : +- Exchange (101) + : : : +- Filter (100) + : : : +- Scan parquet (99) + : : +- Exchange (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Exchange (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -130,560 +124,536 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(8) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(22) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(36) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(46) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(47) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(48) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(49) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(50) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: isnotnull(o_orderkey#X) - -(56) ProjectExecTransformer +(51) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(53) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(55) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(60) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(61) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(62) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(63) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(64) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(71) ProjectExecTransformer +(65) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(67) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(69) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(70) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(71) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(72) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(73) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(74) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(76) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(77) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(78) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(79) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(82) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(83) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(84) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(85) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(86) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(88) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(89) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(90) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(91) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(92) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(93) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(94) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(95) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(97) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(98) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(99) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(100) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(101) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(103) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(104) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(105) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(106) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(107) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(109) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(110) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(115) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(116) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(117) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(118) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(119) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(121) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(122) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(123) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(124) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(125) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(126) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt index 634f26c86f24..0217849e5de2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt @@ -1,31 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (27) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ^ InputAdapter (9) - +- ^ ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (18) + +- ^ SortExecTransformer (16) + +- ^ InputIteratorTransformer (15) + +- ^ InputAdapter (14) + +- ^ ShuffleQueryStage (13), Statistics(X) + +- ColumnarExchange (12) + +- ^ RegularHashAggregateExecTransformer (10) + +- ^ InputIteratorTransformer (9) + +- ^ InputAdapter (8) + +- ^ ShuffleQueryStage (7), Statistics(X) + +- ColumnarExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FlushableHashAggregateExecTransformer (3) + +- ^ ProjectExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (26) + +- Exchange (25) + +- HashAggregate (24) + +- Exchange (23) + +- HashAggregate (22) + +- Project (21) + +- Filter (20) + +- Scan parquet (19) (1) Scan parquet @@ -35,120 +34,116 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(2) FilterExecTransformer -Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -Arguments: (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X, ((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)) AS _pre_X#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(3) FlushableHashAggregateExecTransformer Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, _pre_X#X, _pre_X#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(_pre_X#X), partial_sum(_pre_X#X), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(5) ProjectExecTransformer +(4) ProjectExecTransformer Output [18]: [hash(l_returnflag#X, l_linestatus#X, 42) AS hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(6) WholeStageCodegenTransformer (X) +(5) WholeStageCodegenTransformer (X) Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: false -(7) ColumnarExchange +(6) ColumnarExchange Input [18]: [hash_partition_key#X, l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X], [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(7) ShuffleQueryStage Output [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: X -(9) InputAdapter +(8) InputAdapter Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(10) InputIteratorTransformer +(9) InputIteratorTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(11) RegularHashAggregateExecTransformer +(10) RegularHashAggregateExecTransformer Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(12) WholeStageCodegenTransformer (X) +(11) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(12) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(13) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(14) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(15) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(16) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(18) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(19) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(20) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(21) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(22) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(23) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(24) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(25) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(26) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(27) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt index 873710be9a02..d88b2d72aa24 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt @@ -1,85 +1,81 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (83) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner (47) - :- ^ InputIteratorTransformer (38) - : +- ^ InputAdapter (37) - : +- ^ ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : :- ^ InputIteratorTransformer (23) - : : +- ^ InputAdapter (22) - : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ^ InputAdapter (30) - : +- ^ ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ^ InputAdapter (45) - +- ^ ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (56) + +- TakeOrderedAndProjectExecTransformer (55) + +- ^ ProjectExecTransformer (53) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ^ InputAdapter (50) + +- ^ ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ ShuffledHashJoinExecTransformer Inner (43) + :- ^ InputIteratorTransformer (35) + : +- ^ InputAdapter (34) + : +- ^ ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- ^ ProjectExecTransformer (30) + : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : :- ^ InputIteratorTransformer (21) + : : +- ^ InputAdapter (20) + : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : +- ColumnarExchange (18) + : : +- ^ ProjectExecTransformer (16) + : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : : +- ColumnarExchange (4) + : : : +- ^ ProjectExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (14) + : : +- ^ InputAdapter (13) + : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : +- ColumnarExchange (11) + : : +- ^ ProjectExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (28) + : +- ^ InputAdapter (27) + : +- ^ ShuffleQueryStage (26), Statistics(X) + : +- ColumnarExchange (25) + : +- ^ ProjectExecTransformer (23) + : +- ^ Scan parquet (22) + +- ^ InputIteratorTransformer (42) + +- ^ InputAdapter (41) + +- ^ ShuffleQueryStage (40), Statistics(X) + +- ColumnarExchange (39) + +- ^ ProjectExecTransformer (37) + +- ^ Scan parquet (36) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (82) + +- HashAggregate (81) + +- Exchange (80) + +- HashAggregate (79) + +- Project (78) + +- ShuffledHashJoin Inner BuildRight (77) + :- Exchange (73) + : +- Project (72) + : +- ShuffledHashJoin Inner BuildRight (71) + : :- Exchange (66) + : : +- Project (65) + : : +- ShuffledHashJoin Inner BuildRight (64) + : : :- Exchange (59) + : : : +- Filter (58) + : : : +- Scan parquet (57) + : : +- Exchange (63) + : : +- Project (62) + : : +- Filter (61) + : : +- Scan parquet (60) + : +- Exchange (70) + : +- Project (69) + : +- Filter (68) + : +- Scan parquet (67) + +- Exchange (76) + +- Filter (75) + +- Scan parquet (74) (1) Scan parquet @@ -89,376 +85,360 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [8]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(22) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(36) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(45) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(46) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(48) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(49) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(50) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(51) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(57) ProjectExecTransformer +(53) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(58) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(55) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(57) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(58) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(59) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(60) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(61) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(62) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(63) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(64) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(69) Project +(65) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(66) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(67) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(68) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(69) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(70) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(71) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(76) Project +(72) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(73) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(74) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(75) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(76) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(77) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(78) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(79) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(80) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(81) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(82) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(83) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt index bfe059a11c11..486fe217ad6f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt @@ -1,71 +1,68 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ^ InputAdapter (46) - +- ^ ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ^ InputAdapter (39) - +- ^ ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (47) + +- ^ SortExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ^ InputAdapter (43) + +- ^ ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- ^ FilterExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ^ InputAdapter (36) + +- ^ ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- ^ ProjectExecTransformer (32) + +- ^ FlushableHashAggregateExecTransformer (31) + +- ^ ProjectExecTransformer (30) + +- ^ ShuffledHashJoinExecTransformer Inner (29) + :- ^ InputIteratorTransformer (21) + : +- ^ InputAdapter (20) + : +- ^ ShuffleQueryStage (19), Statistics(X) + : +- ColumnarExchange (18) + : +- ^ ProjectExecTransformer (16) + : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (14) + : +- ^ InputAdapter (13) + : +- ^ ShuffleQueryStage (12), Statistics(X) + : +- ColumnarExchange (11) + : +- ^ ProjectExecTransformer (9) + : +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (28) + +- ^ InputAdapter (27) + +- ^ ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ Scan parquet (22) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (68) + +- Exchange (67) + +- Filter (66) + +- HashAggregate (65) + +- Exchange (64) + +- HashAggregate (63) + +- Project (62) + +- ShuffledHashJoin Inner BuildRight (61) + :- Exchange (56) + : +- Project (55) + : +- ShuffledHashJoin Inner BuildRight (54) + : :- Exchange (50) + : : +- Filter (49) + : : +- Scan parquet (48) + : +- Exchange (53) + : +- Filter (52) + : +- Scan parquet (51) + +- Exchange (60) + +- Project (59) + +- Filter (58) + +- Scan parquet (57) (1) Scan parquet @@ -75,573 +72,556 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(8) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(22) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(31) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(34) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(35) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(36) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(37) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(42) FilterExecTransformer +(39) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(41) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(43) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(45) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(47) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(48) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(49) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(50) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(51) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(52) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(53) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(54) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(58) Project +(55) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(56) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(57) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(58) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(59) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(60) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(61) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(65) Project +(62) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(63) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(64) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(65) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(69) Filter +(66) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(67) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(68) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 39 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (116) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner (96) - :- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner (85) - : :- ^ InputIteratorTransformer (80) - : : +- ^ InputAdapter (79) - : : +- ^ ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ FilterExecTransformer (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ^ InputAdapter (83) - : +- ^ ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ^ InputAdapter (94) - +- ^ ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (98) + +- ^ ProjectExecTransformer (96) + +- ^ RegularHashAggregateExecTransformer (95) + +- ^ RegularHashAggregateExecTransformer (94) + +- ^ ProjectExecTransformer (93) + +- ^ ShuffledHashJoinExecTransformer Inner (92) + :- ^ InputIteratorTransformer (87) + : +- ^ InputAdapter (86) + : +- ^ ShuffleQueryStage (85), Statistics(X) + : +- ColumnarExchange (84) + : +- ^ ProjectExecTransformer (82) + : +- ^ ShuffledHashJoinExecTransformer Inner (81) + : :- ^ InputIteratorTransformer (76) + : : +- ^ InputAdapter (75) + : : +- ^ ShuffleQueryStage (74), Statistics(X) + : : +- ColumnarExchange (73) + : : +- ^ ProjectExecTransformer (71) + : : +- ^ Scan parquet (70) + : +- ^ InputIteratorTransformer (80) + : +- ^ InputAdapter (79) + : +- ^ ShuffleQueryStage (78), Statistics(X) + : +- ReusedExchange (77) + +- ^ InputIteratorTransformer (91) + +- ^ InputAdapter (90) + +- ^ ShuffleQueryStage (89), Statistics(X) + +- ReusedExchange (88) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (115) + +- HashAggregate (114) + +- Project (113) + +- ShuffledHashJoin Inner BuildRight (112) + :- Exchange (107) + : +- Project (106) + : +- ShuffledHashJoin Inner BuildRight (105) + : :- Exchange (101) + : : +- Filter (100) + : : +- Scan parquet (99) + : +- Exchange (104) + : +- Filter (103) + : +- Scan parquet (102) + +- Exchange (111) + +- Project (110) + +- Filter (109) + +- Scan parquet (108) + + +(70) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) FilterExecTransformer -Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -Arguments: isnotnull(ps_suppkey#X) - -(75) ProjectExecTransformer +(71) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(73) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(75) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(77) ReusedExchange [Reuses operator id: 11] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(78) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(79) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(80) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(81) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(86) ProjectExecTransformer +(82) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(84) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(85) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(86) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(87) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(88) ReusedExchange [Reuses operator id: 25] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(89) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(90) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(91) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(92) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(97) ProjectExecTransformer +(93) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(94) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(95) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(100) ProjectExecTransformer +(96) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(101) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(98) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(99) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(100) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(101) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(102) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(103) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(104) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(110) Project +(106) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(107) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(108) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(109) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(110) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(111) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(112) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(113) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(114) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(115) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(116) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt index 85c06695e147..d77bf02f980a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt @@ -1,50 +1,48 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (47) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ RegularHashAggregateExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ ProjectExecTransformer (18) + +- ^ FlushableHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer Inner (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (46) + +- Exchange (45) + +- HashAggregate (44) + +- Exchange (43) + +- HashAggregate (42) + +- Project (41) + +- ShuffledHashJoin Inner BuildLeft (40) + :- Exchange (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Exchange (39) + +- Project (38) + +- Filter (37) + +- Scan parquet (36) (1) Scan parquet @@ -54,210 +52,202 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderpriority#X] -Arguments: isnotnull(o_orderkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(17) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(24) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(26) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(28) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(30) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(33) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(34) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(35) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(36) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(37) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(38) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(39) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(40) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(43) Project +(41) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(43) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(44) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(45) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(46) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(47) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt index c3526c707e4b..796c40d94d57 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt @@ -1,53 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (51) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter (16) + VeloxColumnarToRowExec (35) + +- ^ SortExecTransformer (33) + +- ^ InputIteratorTransformer (32) + +- ^ InputAdapter (31) + +- ^ ShuffleQueryStage (30), Statistics(X) + +- ColumnarExchange (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ^ InputAdapter (25) + +- ^ ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer LeftOuter (15) :- ^ InputIteratorTransformer (7) : +- ^ InputAdapter (6) : +- ^ ShuffleQueryStage (5), Statistics(X) : +- ColumnarExchange (4) : +- ^ ProjectExecTransformer (2) : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (50) + +- Exchange (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- HashAggregate (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftOuter BuildRight (42) + :- Exchange (37) + : +- Scan parquet (36) + +- Exchange (41) + +- Project (40) + +- Filter (39) + +- Scan parquet (38) (1) Scan parquet @@ -85,204 +84,200 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) - -(10) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(13) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(17) ProjectExecTransformer +(16) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(19) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(21) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(25) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(29) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(30) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(31) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(32) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(33) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(36) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(37) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(39) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(40) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(44) Project +(43) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(44) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(46) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(47) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(48) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(49) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(50) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(51) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt index fd8974a5b6cb..180411f0b335 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt @@ -1,38 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (33) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (21) + +- ^ ProjectExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer Inner (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (32) + +- HashAggregate (31) + +- Project (30) + +- ShuffledHashJoin Inner BuildRight (29) + :- Exchange (25) + : +- Project (24) + : +- Filter (23) + : +- Scan parquet (22) + +- Exchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -42,158 +40,150 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(8) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: isnotnull(p_partkey#X) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) ProjectExecTransformer +(19) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(22) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(21) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(22) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(23) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(24) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(25) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(26) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(27) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(28) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(29) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(32) Project +(30) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(31) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(32) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(33) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt index 8fd855a59c42..84df2af7dd53 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt @@ -1,45 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (40) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner (21) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ^ InputAdapter (17) - +- ^ ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- AQEShuffleRead (24) + +- ShuffleQueryStage (23), Statistics(X) + +- ColumnarExchange (22) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner (19) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ InputIteratorTransformer (16) + +- ^ InputAdapter (15) + +- ^ ShuffleQueryStage (14), Statistics(X) + +- ColumnarExchange (13) + +- ^ ProjectExecTransformer (11) + +- ^ FlushableHashAggregateExecTransformer (10) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (39) + +- Exchange (38) + +- Project (37) + +- ShuffledHashJoin Inner BuildLeft (36) + :- Exchange (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Filter (35) + +- HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -49,343 +47,330 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -Arguments: isnotnull(s_suppkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(10) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(11) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(13) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(14) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(15) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(16) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(20) FilterExecTransformer +(18) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(22) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(22) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(23) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(24) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(26) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(27) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(28) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(29) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(30) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(31) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(32) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(33) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(37) Filter +(35) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(36) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(39) Project +(37) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(38) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(39) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(40) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 18 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (64) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ^ InputAdapter (51) - +- ^ ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ FilterExecTransformer (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (55) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ ProjectExecTransformer (51) + +- ^ RegularHashAggregateExecTransformer (50) + +- ^ InputIteratorTransformer (49) + +- ^ InputAdapter (48) + +- ^ ShuffleQueryStage (47), Statistics(X) + +- ColumnarExchange (46) + +- ^ ProjectExecTransformer (44) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ Scan parquet (41) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- Filter (57) + +- Scan parquet (56) -(43) Scan parquet +(41) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) FilterExecTransformer -Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) - -(45) ProjectExecTransformer +(42) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(44) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(46) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(47) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(48) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(49) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) ProjectExecTransformer +(51) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(55) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(56) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(57) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(58) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(59) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(60) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(61) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(65) HashAggregate +(62) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(63) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(64) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt index 498c1e83e15b..456656af6264 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt @@ -1,64 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (62) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ^ InputAdapter (38) - +- ^ ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ^ InputAdapter (32) - +- ^ ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (40) + +- ^ SortExecTransformer (38) + +- ^ InputIteratorTransformer (37) + +- ^ InputAdapter (36) + +- ^ ShuffleQueryStage (35), Statistics(X) + +- ColumnarExchange (34) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ InputIteratorTransformer (31) + +- ^ InputAdapter (30) + +- ^ ShuffleQueryStage (29), Statistics(X) + +- ColumnarExchange (28) + +- ^ ProjectExecTransformer (26) + +- ^ FlushableHashAggregateExecTransformer (25) + +- ^ RegularHashAggregateExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ ProjectExecTransformer (18) + +- ^ FlushableHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer Inner (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (61) + +- Exchange (60) + +- HashAggregate (59) + +- Exchange (58) + +- HashAggregate (57) + +- HashAggregate (56) + +- Exchange (55) + +- HashAggregate (54) + +- Project (53) + +- ShuffledHashJoin Inner BuildRight (52) + :- Exchange (48) + : +- BroadcastHashJoin LeftAnti BuildRight (47) + : :- Filter (42) + : : +- Scan parquet (41) + : +- BroadcastExchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Exchange (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -68,285 +66,277 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [ps_partkey#X, ps_suppkey#X] -Arguments: isnotnull(ps_partkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Input [2]: [ps_partkey#X, ps_suppkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(17) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(24) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(25) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(26) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(27) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(28) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(29) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(30) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(31) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(34) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(36) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(38) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(40) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(41) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(42) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(43) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(44) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(45) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(46) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(50) Exchange +(48) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(49) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(50) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(51) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(52) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(55) Project +(53) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(54) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(56) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(57) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(58) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(59) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(61) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(62) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt index f15f42b69cd8..276e3e66628e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt @@ -1,59 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ^ InputAdapter (26) - +- ^ ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ ProjectExecTransformer (29) + +- ^ ShuffledHashJoinExecTransformer Inner (28) + :- ^ ProjectExecTransformer (16) + : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (14) + : +- ^ InputAdapter (13) + : +- ^ ShuffleQueryStage (12), Statistics(X) + : +- ColumnarExchange (11) + : +- ^ ProjectExecTransformer (9) + : +- ^ Scan parquet (8) + +- ^ FilterExecTransformer (27) + +- ^ ProjectExecTransformer (26) + +- ^ RegularHashAggregateExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ^ InputAdapter (23) + +- ^ ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- ^ ProjectExecTransformer (19) + +- ^ FlushableHashAggregateExecTransformer (18) + +- ^ Scan parquet (17) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (53) + +- HashAggregate (52) + +- Project (51) + +- ShuffledHashJoin Inner BuildRight (50) + :- Project (43) + : +- ShuffledHashJoin Inner BuildRight (42) + : :- Exchange (37) + : : +- Filter (36) + : : +- Scan parquet (35) + : +- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Filter (49) + +- HashAggregate (48) + +- Exchange (47) + +- HashAggregate (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -63,266 +60,254 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -Arguments: (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [p_partkey#X, p_brand#X, p_container#X] -Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(17) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) FilterExecTransformer -Input [2]: [l_partkey#X, l_quantity#X] -Arguments: isnotnull(l_partkey#X) - -(21) FlushableHashAggregateExecTransformer +(18) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(19) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(20) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(21) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(23) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(25) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(26) ProjectExecTransformer Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(27) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(32) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(34) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(35) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(36) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(37) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(38) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(39) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(40) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(41) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(46) Project +(43) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(44) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(45) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(46) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(47) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(48) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(49) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(50) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(51) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(52) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(53) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt index ad97ed4013f0..2bda86546324 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt @@ -1,96 +1,93 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner (58) - :- ^ InputIteratorTransformer (41) - : +- ^ InputAdapter (40) - : +- ^ ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner (35) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ^ InputAdapter (33) - : +- ^ ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (28) - : :- ^ InputIteratorTransformer (16) - : : +- ^ InputAdapter (15) - : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ^ InputAdapter (23) - : +- ^ ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (57) - :- ^ InputIteratorTransformer (49) - : +- ^ InputAdapter (48) - : +- ^ ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ^ InputAdapter (52) - +- ^ ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (61) + +- TakeOrderedAndProjectExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ ShuffledHashJoinExecTransformer Inner (55) + :- ^ InputIteratorTransformer (39) + : +- ^ InputAdapter (38) + : +- ^ ShuffleQueryStage (37), Statistics(X) + : +- ColumnarExchange (36) + : +- ^ ProjectExecTransformer (34) + : +- ^ ShuffledHashJoinExecTransformer Inner (33) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (32) + : +- ^ InputAdapter (31) + : +- ^ ShuffleQueryStage (30), Statistics(X) + : +- ColumnarExchange (29) + : +- ^ ProjectExecTransformer (27) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (26) + : :- ^ InputIteratorTransformer (14) + : : +- ^ InputAdapter (13) + : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : +- ColumnarExchange (11) + : : +- ^ ProjectExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ ProjectExecTransformer (25) + : +- ^ FilterExecTransformer (24) + : +- ^ RegularHashAggregateExecTransformer (23) + : +- ^ InputIteratorTransformer (22) + : +- ^ InputAdapter (21) + : +- ^ ShuffleQueryStage (20), Statistics(X) + : +- ColumnarExchange (19) + : +- ^ ProjectExecTransformer (17) + : +- ^ FlushableHashAggregateExecTransformer (16) + : +- ^ Scan parquet (15) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (54) + :- ^ InputIteratorTransformer (46) + : +- ^ InputAdapter (45) + : +- ^ ShuffleQueryStage (44), Statistics(X) + : +- ColumnarExchange (43) + : +- ^ ProjectExecTransformer (41) + : +- ^ Scan parquet (40) + +- ^ ProjectExecTransformer (53) + +- ^ FilterExecTransformer (52) + +- ^ RegularHashAggregateExecTransformer (51) + +- ^ InputIteratorTransformer (50) + +- ^ InputAdapter (49) + +- ^ ShuffleQueryStage (48), Statistics(X) + +- ReusedExchange (47) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (93) + +- HashAggregate (92) + +- HashAggregate (91) + +- Project (90) + +- ShuffledHashJoin Inner BuildRight (89) + :- Exchange (78) + : +- Project (77) + : +- ShuffledHashJoin Inner BuildLeft (76) + : :- Exchange (64) + : : +- Filter (63) + : : +- Scan parquet (62) + : +- Exchange (75) + : +- ShuffledHashJoin LeftSemi BuildRight (74) + : :- Exchange (67) + : : +- Filter (66) + : : +- Scan parquet (65) + : +- Project (73) + : +- Filter (72) + : +- HashAggregate (71) + : +- Exchange (70) + : +- HashAggregate (69) + : +- Scan parquet (68) + +- ShuffledHashJoin LeftSemi BuildRight (88) + :- Exchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- Project (87) + +- Filter (86) + +- HashAggregate (85) + +- Exchange (84) + +- HashAggregate (83) + +- Scan parquet (82) (1) Scan parquet @@ -100,440 +97,428 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_name#X] -Arguments: isnotnull(c_custkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X] Input [2]: [c_custkey#X, c_name#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(15) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(16) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(17) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(18) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(19) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(20) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(21) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(22) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(24) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(25) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(26) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(29) ProjectExecTransformer +(27) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(29) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(30) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(31) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(32) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(36) ProjectExecTransformer +(34) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(36) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(37) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(38) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(39) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(40) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) FilterExecTransformer -Input [2]: [l_orderkey#X, l_quantity#X] -Arguments: isnotnull(l_orderkey#X) - -(44) ProjectExecTransformer +(41) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(43) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(44) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(45) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(46) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(47) ReusedExchange [Reuses operator id: 19] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(48) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(49) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(50) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(52) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(53) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(54) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(58) ShuffledHashJoinExecTransformer +(55) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(59) ProjectExecTransformer +(56) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(60) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(62) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(63) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(64) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(65) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(66) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(67) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(68) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(69) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(70) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(71) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(72) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(73) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(78) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(76) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(80) Project +(77) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(78) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(79) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(80) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(81) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(82) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(83) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(84) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(85) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(86) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(87) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(88) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(92) ShuffledHashJoin +(89) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(93) Project +(90) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(91) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(92) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(93) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt index 4b5c20d3a1b1..cff613274b00 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt @@ -1,37 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (32) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ RegularHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer Inner (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (31) + +- HashAggregate (30) + +- Project (29) + +- ShuffledHashJoin Inner BuildRight (28) + :- Exchange (24) + : +- Project (23) + : +- Filter (22) + : +- Scan parquet (21) + +- Exchange (27) + +- Filter (26) + +- Scan parquet (25) (1) Scan parquet @@ -41,154 +39,146 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(2) FilterExecTransformer -Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -Arguments: ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(17) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(18) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(21) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(22) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(23) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(24) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(25) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(26) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(27) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(28) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(29) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(30) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(31) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(32) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt index 17f48f8ffa72..541b0f0aebab 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt @@ -1,119 +1,114 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (118) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (62) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) - : :- ^ InputIteratorTransformer (31) - : : +- ^ InputAdapter (30) - : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ^ InputAdapter (15) - : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ^ InputAdapter (23) - : : +- ^ ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi (44) - : :- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ^ InputAdapter (42) - : +- ^ ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (78) + +- AQEShuffleRead (77) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- ^ ProjectExecTransformer (73) + +- ^ ShuffledHashJoinExecTransformer Inner (72) + :- ^ InputIteratorTransformer (64) + : +- ^ InputAdapter (63) + : +- ^ ShuffleQueryStage (62), Statistics(X) + : +- ColumnarExchange (61) + : +- ^ ProjectExecTransformer (59) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (58) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (57) + : +- ^ InputAdapter (56) + : +- ^ ShuffleQueryStage (55), Statistics(X) + : +- ColumnarExchange (54) + : +- ^ ProjectExecTransformer (52) + : +- ^ ShuffledHashJoinExecTransformer Inner (51) + : :- ^ InputIteratorTransformer (28) + : : +- ^ InputAdapter (27) + : : +- ^ ShuffleQueryStage (26), Statistics(X) + : : +- ColumnarExchange (25) + : : +- ^ ProjectExecTransformer (23) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi (22) + : : :- ^ InputIteratorTransformer (14) + : : : +- ^ InputAdapter (13) + : : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : : +- ColumnarExchange (11) + : : : +- ^ ProjectExecTransformer (9) + : : : +- ^ Scan parquet (8) + : : +- ^ InputIteratorTransformer (21) + : : +- ^ InputAdapter (20) + : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : +- ColumnarExchange (18) + : : +- ^ ProjectExecTransformer (16) + : : +- ^ Scan parquet (15) + : +- ^ InputIteratorTransformer (50) + : +- ^ InputAdapter (49) + : +- ^ ShuffleQueryStage (48), Statistics(X) + : +- ColumnarExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ FilterExecTransformer (44) + : +- ^ ProjectExecTransformer (43) + : +- ^ RegularHashAggregateExecTransformer (42) + : +- ^ RegularHashAggregateExecTransformer (41) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi (40) + : :- ^ InputIteratorTransformer (35) + : : +- ^ InputAdapter (34) + : : +- ^ ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ Scan parquet (29) + : +- ^ InputIteratorTransformer (39) + : +- ^ InputAdapter (38) + : +- ^ ShuffleQueryStage (37), Statistics(X) + : +- ReusedExchange (36) + +- ^ InputIteratorTransformer (71) + +- ^ InputAdapter (70) + +- ^ ShuffleQueryStage (69), Statistics(X) + +- ColumnarExchange (68) + +- ^ ProjectExecTransformer (66) + +- ^ Scan parquet (65) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (117) + +- Exchange (116) + +- Project (115) + +- ShuffledHashJoin Inner BuildRight (114) + :- Exchange (109) + : +- Project (108) + : +- ShuffledHashJoin LeftSemi BuildRight (107) + : :- Exchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- Exchange (106) + : +- Project (105) + : +- ShuffledHashJoin Inner BuildLeft (104) + : :- Exchange (90) + : : +- ShuffledHashJoin LeftSemi BuildRight (89) + : : :- Exchange (84) + : : : +- Filter (83) + : : : +- Scan parquet (82) + : : +- Exchange (88) + : : +- Project (87) + : : +- Filter (86) + : : +- Scan parquet (85) + : +- Exchange (103) + : +- Filter (102) + : +- HashAggregate (101) + : +- HashAggregate (100) + : +- ShuffledHashJoin LeftSemi BuildRight (99) + : :- Exchange (94) + : : +- Project (93) + : : +- Filter (92) + : : +- Scan parquet (91) + : +- Exchange (98) + : +- Project (97) + : +- Filter (96) + : +- Scan parquet (95) + +- Exchange (113) + +- Project (112) + +- Filter (111) + +- Scan parquet (110) (1) Scan parquet @@ -123,530 +118,510 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -Arguments: isnotnull(s_nationkey#X) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [5]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(15) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) - -(19) ProjectExecTransformer +(16) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(18) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(19) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(20) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(21) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(29) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) FilterExecTransformer -Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) - -(34) ProjectExecTransformer +(30) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(34) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(36) ReusedExchange [Reuses operator id: 18] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(37) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(38) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(39) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(40) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(45) RegularHashAggregateExecTransformer +(41) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(42) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(43) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(44) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(47) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(48) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(49) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(50) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(51) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(52) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(54) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(55) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(56) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(57) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(58) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(63) ProjectExecTransformer +(59) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(61) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(62) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(63) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(64) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) - -(71) ProjectExecTransformer +(66) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(68) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(69) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(70) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(71) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(72) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(73) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(74) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(75) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(76) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(77) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(78) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(79) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(80) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(81) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(82) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(83) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(84) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(85) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(86) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(87) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(88) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(89) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(95) Exchange +(90) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(91) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(92) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(93) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(94) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(95) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(96) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(97) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(98) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(99) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(105) HashAggregate +(100) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(101) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(102) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(103) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(105) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(106) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(113) Project +(108) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(109) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(110) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(111) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(112) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(113) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(120) Project +(115) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(116) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(117) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(118) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt index 7b5d71ec6d26..58be95c2bc00 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt @@ -1,113 +1,108 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (113) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ^ InputAdapter (77) - +- ^ ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner (70) - :- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner (55) - : :- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ^ InputAdapter (7) - : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ^ InputAdapter (38) - : : +- ^ ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ^ InputAdapter (22) - : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ^ InputAdapter (31) - : : +- ^ ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ^ InputAdapter (53) - : +- ^ ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ^ InputAdapter (68) - +- ^ ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (76) + +- ^ RegularHashAggregateExecTransformer (74) + +- ^ InputIteratorTransformer (73) + +- ^ InputAdapter (72) + +- ^ ShuffleQueryStage (71), Statistics(X) + +- ColumnarExchange (70) + +- ^ ProjectExecTransformer (68) + +- ^ FlushableHashAggregateExecTransformer (67) + +- ^ ProjectExecTransformer (66) + +- ^ ShuffledHashJoinExecTransformer Inner (65) + :- ^ InputIteratorTransformer (57) + : +- ^ InputAdapter (56) + : +- ^ ShuffleQueryStage (55), Statistics(X) + : +- ColumnarExchange (54) + : +- ^ ProjectExecTransformer (52) + : +- ^ ShuffledHashJoinExecTransformer Inner (51) + : :- ^ InputIteratorTransformer (43) + : : +- ^ InputAdapter (42) + : : +- ^ ShuffleQueryStage (41), Statistics(X) + : : +- ColumnarExchange (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer Inner (37) + : : :- ^ InputIteratorTransformer (7) + : : : +- ^ InputAdapter (6) + : : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : : +- ColumnarExchange (4) + : : : +- ^ ProjectExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (36) + : : +- ^ InputAdapter (35) + : : +- ^ ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- ^ ProjectExecTransformer (31) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti (30) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi (22) + : : : :- ^ InputIteratorTransformer (14) + : : : : +- ^ InputAdapter (13) + : : : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : : : +- ColumnarExchange (11) + : : : : +- ^ ProjectExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (21) + : : : +- ^ InputAdapter (20) + : : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : : +- ColumnarExchange (18) + : : : +- ^ ProjectExecTransformer (16) + : : : +- ^ Scan parquet (15) + : : +- ^ InputIteratorTransformer (29) + : : +- ^ InputAdapter (28) + : : +- ^ ShuffleQueryStage (27), Statistics(X) + : : +- ColumnarExchange (26) + : : +- ^ ProjectExecTransformer (24) + : : +- ^ Scan parquet (23) + : +- ^ InputIteratorTransformer (50) + : +- ^ InputAdapter (49) + : +- ^ ShuffleQueryStage (48), Statistics(X) + : +- ColumnarExchange (47) + : +- ^ ProjectExecTransformer (45) + : +- ^ Scan parquet (44) + +- ^ InputIteratorTransformer (64) + +- ^ InputAdapter (63) + +- ^ ShuffleQueryStage (62), Statistics(X) + +- ColumnarExchange (61) + +- ^ ProjectExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (112) + +- HashAggregate (111) + +- Exchange (110) + +- HashAggregate (109) + +- Project (108) + +- ShuffledHashJoin Inner BuildRight (107) + :- Exchange (102) + : +- Project (101) + : +- ShuffledHashJoin Inner BuildRight (100) + : :- Exchange (95) + : : +- Project (94) + : : +- ShuffledHashJoin Inner BuildLeft (93) + : : :- Exchange (79) + : : : +- Filter (78) + : : : +- Scan parquet (77) + : : +- Exchange (92) + : : +- ShuffledHashJoin LeftAnti BuildRight (91) + : : :- ShuffledHashJoin LeftSemi BuildRight (86) + : : : :- Exchange (83) + : : : : +- Project (82) + : : : : +- Filter (81) + : : : : +- Scan parquet (80) + : : : +- Exchange (85) + : : : +- Scan parquet (84) + : : +- Exchange (90) + : : +- Project (89) + : : +- Filter (88) + : : +- Scan parquet (87) + : +- Exchange (99) + : +- Project (98) + : +- Filter (97) + : +- Scan parquet (96) + +- Exchange (106) + +- Project (105) + +- Filter (104) + +- Scan parquet (103) (1) Scan parquet @@ -117,512 +112,492 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(15) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(23) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) - -(27) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(28) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(31) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(41) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(40) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(41) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(42) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(43) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(44) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderstatus#X] -Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) - -(49) ProjectExecTransformer +(45) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(47) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(48) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(49) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(50) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(51) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(56) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(54) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(55) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(56) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(57) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(58) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) - -(64) ProjectExecTransformer +(59) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(61) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(62) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(63) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(64) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(65) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(71) ProjectExecTransformer +(66) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(67) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(68) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(69) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(70) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(71) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(72) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(73) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(74) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(76) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(77) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(78) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(79) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(80) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(81) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(82) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(83) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(84) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(85) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(86) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(87) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(88) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(89) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(90) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(91) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(92) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(93) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(99) Project +(94) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(95) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(96) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(97) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(98) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(99) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(106) Project +(101) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(102) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(103) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(104) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(105) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(106) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(113) Project +(108) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(109) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(110) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(111) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(112) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(113) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt index 86f293b4ff07..5a5cb9c96758 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt @@ -1,47 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ^ InputAdapter (29) - +- ^ ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ^ InputAdapter (23) - +- ^ ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti (16) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ^ InputAdapter (14) - +- ^ ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ RegularHashAggregateExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ ProjectExecTransformer (18) + +- ^ FlushableHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer LeftAnti (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- ShuffledHashJoin LeftAnti BuildRight (38) + :- Exchange (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Exchange (37) + +- Scan parquet (36) (1) Scan parquet @@ -51,332 +50,302 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [4]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(8) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(9) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(11) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(12) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(13) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(14) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(17) ProjectExecTransformer +(16) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(17) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(18) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(20) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(21) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(22) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(23) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(24) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(26) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(28) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(30) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(33) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(34) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(35) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(36) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(37) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(38) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(40) Project +(39) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(40) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(41) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(42) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(43) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(44) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (56) + +- ^ RegularHashAggregateExecTransformer (54) + +- ^ InputIteratorTransformer (53) + +- ^ InputAdapter (52) + +- ^ ShuffleQueryStage (51), Statistics(X) + +- ColumnarExchange (50) + +- ^ FlushableHashAggregateExecTransformer (48) + +- ^ ProjectExecTransformer (47) + +- ^ Scan parquet (46) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (62) + +- Exchange (61) + +- HashAggregate (60) + +- Project (59) + +- Filter (58) + +- Scan parquet (57) -(47) Scan parquet +(46) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) FilterExecTransformer -Input [2]: [c_phone#X, c_acctbal#X] -Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) - -(49) ProjectExecTransformer +(47) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(48) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(50) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(51) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(52) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(53) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(54) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(56) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(57) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(58) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(59) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(60) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(61) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(62) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] -Arguments: isFinalPlan=true - -Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) -+- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ^ InputAdapter (54) - +- ^ ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) -+- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) \ No newline at end of file +Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt index 0c75b0257aba..25e019dc37ba 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt @@ -1,60 +1,57 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (56) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner (32) - :- ^ InputIteratorTransformer (23) - : +- ^ InputAdapter (22) - : +- ^ ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : :- ^ InputIteratorTransformer (8) - : : +- ^ InputAdapter (7) - : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ^ InputAdapter (15) - : +- ^ ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (36) + +- TakeOrderedAndProjectExecTransformer (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ ProjectExecTransformer (30) + +- ^ ShuffledHashJoinExecTransformer Inner (29) + :- ^ InputIteratorTransformer (21) + : +- ^ InputAdapter (20) + : +- ^ ShuffleQueryStage (19), Statistics(X) + : +- ColumnarExchange (18) + : +- ^ ProjectExecTransformer (16) + : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : :- ^ InputIteratorTransformer (7) + : : +- ^ InputAdapter (6) + : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : +- ColumnarExchange (4) + : : +- ^ ProjectExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (14) + : +- ^ InputAdapter (13) + : +- ^ ShuffleQueryStage (12), Statistics(X) + : +- ColumnarExchange (11) + : +- ^ ProjectExecTransformer (9) + : +- ^ Scan parquet (8) + +- ^ InputIteratorTransformer (28) + +- ^ InputAdapter (27) + +- ^ ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ Scan parquet (22) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (55) + +- HashAggregate (54) + +- HashAggregate (53) + +- Project (52) + +- ShuffledHashJoin Inner BuildRight (51) + :- Exchange (46) + : +- Project (45) + : +- ShuffledHashJoin Inner BuildLeft (44) + : :- Exchange (40) + : : +- Project (39) + : : +- Filter (38) + : : +- Scan parquet (37) + : +- Exchange (43) + : +- Filter (42) + : +- Scan parquet (41) + +- Exchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -64,260 +61,248 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_mktsegment#X] -Arguments: ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [2]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(8) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(22) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(36) ProjectExecTransformer +(33) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(37) WholeStageCodegenTransformer (X) +(34) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(35) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(37) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(38) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(39) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(40) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(41) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(42) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(43) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(44) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(48) Project +(45) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(46) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(47) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(48) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(49) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(50) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(55) Project +(52) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(53) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(54) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(55) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(56) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt index ee47fca61115..f57e74a85df5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt @@ -1,51 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (48) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ^ InputAdapter (30) - +- ^ ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ^ InputAdapter (24) - +- ^ ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi (17) - :- ^ InputIteratorTransformer (8) - : +- ^ InputAdapter (7) - : +- ^ ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ^ InputAdapter (15) - +- ^ ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ^ InputAdapter (28) + +- ^ ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ RegularHashAggregateExecTransformer (24) + +- ^ InputIteratorTransformer (23) + +- ^ InputAdapter (22) + +- ^ ShuffleQueryStage (21), Statistics(X) + +- ColumnarExchange (20) + +- ^ ProjectExecTransformer (18) + +- ^ FlushableHashAggregateExecTransformer (17) + +- ^ ProjectExecTransformer (16) + +- ^ ShuffledHashJoinExecTransformer LeftSemi (15) + :- ^ InputIteratorTransformer (7) + : +- ^ InputAdapter (6) + : +- ^ ShuffleQueryStage (5), Statistics(X) + : +- ColumnarExchange (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (14) + +- ^ InputAdapter (13) + +- ^ ShuffleQueryStage (12), Statistics(X) + +- ColumnarExchange (11) + +- ^ ProjectExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (47) + +- Exchange (46) + +- HashAggregate (45) + +- Exchange (44) + +- HashAggregate (43) + +- Project (42) + +- ShuffledHashJoin LeftSemi BuildRight (41) + :- Exchange (36) + : +- Project (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Exchange (40) + +- Project (39) + +- Filter (38) + +- Scan parquet (37) (1) Scan parquet @@ -55,214 +53,206 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(2) FilterExecTransformer -Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -Arguments: ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(17) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(18) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(20) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(21) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(22) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(23) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(24) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(26) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(28) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(30) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(33) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(34) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(35) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(36) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(37) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(38) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(39) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(40) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(41) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(44) Project +(42) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(43) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(44) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(45) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(46) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(47) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(48) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt index 39a8f483f639..9de96b917fad 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt @@ -1,127 +1,121 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (88) + +- ^ SortExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ^ InputAdapter (78) + +- ^ ShuffleQueryStage (77), Statistics(X) + +- ColumnarExchange (76) + +- ^ ProjectExecTransformer (74) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ ShuffledHashJoinExecTransformer Inner (71) + :- ^ InputIteratorTransformer (63) + : +- ^ InputAdapter (62) + : +- ^ ShuffleQueryStage (61), Statistics(X) + : +- ColumnarExchange (60) + : +- ^ ProjectExecTransformer (58) + : +- ^ ShuffledHashJoinExecTransformer Inner (57) + : :- ^ InputIteratorTransformer (49) + : : +- ^ InputAdapter (48) + : : +- ^ ShuffleQueryStage (47), Statistics(X) + : : +- ColumnarExchange (46) + : : +- ^ ProjectExecTransformer (44) + : : +- ^ ShuffledHashJoinExecTransformer Inner (43) + : : :- ^ InputIteratorTransformer (35) + : : : +- ^ InputAdapter (34) + : : : +- ^ ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- ^ ProjectExecTransformer (30) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : : : :- ^ InputIteratorTransformer (21) + : : : : +- ^ InputAdapter (20) + : : : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : : : +- ColumnarExchange (18) + : : : : +- ^ ProjectExecTransformer (16) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : : : : +- ColumnarExchange (4) + : : : : : +- ^ ProjectExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (14) + : : : : +- ^ InputAdapter (13) + : : : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : : : +- ColumnarExchange (11) + : : : : +- ^ ProjectExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (28) + : : : +- ^ InputAdapter (27) + : : : +- ^ ShuffleQueryStage (26), Statistics(X) + : : : +- ColumnarExchange (25) + : : : +- ^ ProjectExecTransformer (23) + : : : +- ^ Scan parquet (22) + : : +- ^ InputIteratorTransformer (42) + : : +- ^ InputAdapter (41) + : : +- ^ ShuffleQueryStage (40), Statistics(X) + : : +- ColumnarExchange (39) + : : +- ^ ProjectExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (56) + : +- ^ InputAdapter (55) + : +- ^ ShuffleQueryStage (54), Statistics(X) + : +- ColumnarExchange (53) + : +- ^ ProjectExecTransformer (51) + : +- ^ Scan parquet (50) + +- ^ InputIteratorTransformer (70) + +- ^ InputAdapter (69) + +- ^ ShuffleQueryStage (68), Statistics(X) + +- ColumnarExchange (67) + +- ^ ProjectExecTransformer (65) + +- ^ Scan parquet (64) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (127) + +- Exchange (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildRight (108) + : : :- Exchange (104) + : : : +- Project (103) + : : : +- ShuffledHashJoin Inner BuildRight (102) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- ShuffledHashJoin Inner BuildLeft (96) + : : : : :- Exchange (91) + : : : : : +- Filter (90) + : : : : : +- Scan parquet (89) + : : : : +- Exchange (95) + : : : : +- Project (94) + : : : : +- Filter (93) + : : : : +- Scan parquet (92) + : : : +- Exchange (101) + : : : +- Filter (100) + : : : +- Scan parquet (99) + : : +- Exchange (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Exchange (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -131,574 +125,550 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(8) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(22) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(36) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(46) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(47) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(48) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(49) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(50) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) FilterExecTransformer -Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(56) ProjectExecTransformer +(51) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(53) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(54) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(55) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(56) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(60) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(61) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(62) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(63) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(64) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) - -(71) ProjectExecTransformer +(65) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(67) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(68) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(69) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(70) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(71) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(72) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(73) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(74) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(76) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(77) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(78) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(79) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(82) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(83) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(84) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(85) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(86) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(88) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(89) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(90) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(91) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(92) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(93) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(94) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(95) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(103) Project +(97) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(98) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(99) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(100) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(101) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(109) Project +(103) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(104) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(105) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(106) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(107) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(115) Project +(109) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(110) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(111) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(112) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(113) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(121) Project +(115) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(116) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(117) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(118) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(119) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(120) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(128) Project +(122) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(123) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(124) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(125) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(132) Exchange +(126) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(127) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt index 0882ff9e151c..733adf0d0b4a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt @@ -1,23 +1,22 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (18) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ^ InputAdapter (8) - +- ^ ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (11) + +- ^ RegularHashAggregateExecTransformer (9) + +- ^ InputIteratorTransformer (8) + +- ^ InputAdapter (7) + +- ^ ShuffleQueryStage (6), Statistics(X) + +- ColumnarExchange (5) + +- ^ FlushableHashAggregateExecTransformer (3) + +- ^ ProjectExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (17) + +- Exchange (16) + +- HashAggregate (15) + +- Project (14) + +- Filter (13) + +- Scan parquet (12) (1) Scan parquet @@ -27,86 +26,82 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(2) FilterExecTransformer -Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(3) FlushableHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(5) WholeStageCodegenTransformer (X) +(4) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(5) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(6) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(7) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(8) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(9) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(11) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(12) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(13) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(14) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(15) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(16) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(17) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(18) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt index 29c3f048b7f8..50c9b4a231f1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt @@ -1,122 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (123) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ^ InputAdapter (86) - +- ^ ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ^ InputAdapter (80) - +- ^ ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner (73) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ^ InputAdapter (71) - +- ^ ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ^ InputAdapter (81) + +- ^ ShuffleQueryStage (80), Statistics(X) + +- ColumnarExchange (79) + +- ^ RegularHashAggregateExecTransformer (77) + +- ^ InputIteratorTransformer (76) + +- ^ InputAdapter (75) + +- ^ ShuffleQueryStage (74), Statistics(X) + +- ColumnarExchange (73) + +- ^ ProjectExecTransformer (71) + +- ^ FlushableHashAggregateExecTransformer (70) + +- ^ ProjectExecTransformer (69) + +- ^ ShuffledHashJoinExecTransformer Inner (68) + :- ^ InputIteratorTransformer (63) + : +- ^ InputAdapter (62) + : +- ^ ShuffleQueryStage (61), Statistics(X) + : +- ColumnarExchange (60) + : +- ^ ProjectExecTransformer (58) + : +- ^ ShuffledHashJoinExecTransformer Inner (57) + : :- ^ InputIteratorTransformer (49) + : : +- ^ InputAdapter (48) + : : +- ^ ShuffleQueryStage (47), Statistics(X) + : : +- ColumnarExchange (46) + : : +- ^ ProjectExecTransformer (44) + : : +- ^ ShuffledHashJoinExecTransformer Inner (43) + : : :- ^ InputIteratorTransformer (35) + : : : +- ^ InputAdapter (34) + : : : +- ^ ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- ^ ProjectExecTransformer (30) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : : : :- ^ InputIteratorTransformer (21) + : : : : +- ^ InputAdapter (20) + : : : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : : : +- ColumnarExchange (18) + : : : : +- ^ ProjectExecTransformer (16) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : : : : +- ColumnarExchange (4) + : : : : : +- ^ ProjectExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (14) + : : : : +- ^ InputAdapter (13) + : : : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : : : +- ColumnarExchange (11) + : : : : +- ^ ProjectExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (28) + : : : +- ^ InputAdapter (27) + : : : +- ^ ShuffleQueryStage (26), Statistics(X) + : : : +- ColumnarExchange (25) + : : : +- ^ ProjectExecTransformer (23) + : : : +- ^ Scan parquet (22) + : : +- ^ InputIteratorTransformer (42) + : : +- ^ InputAdapter (41) + : : +- ^ ShuffleQueryStage (40), Statistics(X) + : : +- ColumnarExchange (39) + : : +- ^ ProjectExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (56) + : +- ^ InputAdapter (55) + : +- ^ ShuffleQueryStage (54), Statistics(X) + : +- ColumnarExchange (53) + : +- ^ ProjectExecTransformer (51) + : +- ^ Scan parquet (50) + +- ^ InputIteratorTransformer (67) + +- ^ InputAdapter (66) + +- ^ ShuffleQueryStage (65), Statistics(X) + +- ReusedExchange (64) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (122) + +- Exchange (121) + +- HashAggregate (120) + +- Exchange (119) + +- HashAggregate (118) + +- Project (117) + +- ShuffledHashJoin Inner BuildRight (116) + :- Exchange (112) + : +- Project (111) + : +- ShuffledHashJoin Inner BuildRight (110) + : :- Exchange (106) + : : +- Project (105) + : : +- ShuffledHashJoin Inner BuildRight (104) + : : :- Exchange (100) + : : : +- Project (99) + : : : +- ShuffledHashJoin Inner BuildRight (98) + : : : :- Exchange (94) + : : : : +- Project (93) + : : : : +- ShuffledHashJoin Inner BuildLeft (92) + : : : : :- Exchange (88) + : : : : : +- Filter (87) + : : : : : +- Scan parquet (86) + : : : : +- Exchange (91) + : : : : +- Filter (90) + : : : : +- Scan parquet (89) + : : : +- Exchange (97) + : : : +- Filter (96) + : : : +- Scan parquet (95) + : : +- Exchange (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Exchange (109) + : +- Filter (108) + : +- Scan parquet (107) + +- Exchange (115) + +- Filter (114) + +- Scan parquet (113) (1) Scan parquet @@ -126,546 +121,526 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(22) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [2]: [o_orderkey#X, o_custkey#X] -Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(46) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(47) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(48) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(49) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(50) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) - -(56) ProjectExecTransformer +(51) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(53) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(55) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(60) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(61) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(62) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(63) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(64) ReusedExchange [Reuses operator id: 53] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(65) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(66) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(67) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(68) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(69) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(70) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(73) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(74) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(75) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(76) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(77) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(78) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(79) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(80) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(81) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(82) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(83) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(86) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(87) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(88) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(89) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(90) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(91) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(92) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(98) Project +(93) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(94) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(95) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(96) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(97) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(98) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(104) Project +(99) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(100) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(101) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(102) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(103) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(110) Project +(105) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(106) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(107) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(108) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(109) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(110) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(116) Project +(111) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(112) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(113) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(114) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(115) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(117) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(118) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(119) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(120) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(121) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(122) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(123) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt index 9c24e9ec5a01..5ed8ee8b6b4f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt @@ -1,166 +1,158 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (169) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ^ InputAdapter (121) - +- ^ ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ^ InputAdapter (114) - +- ^ ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner (107) - :- ^ InputIteratorTransformer (98) - : +- ^ InputAdapter (97) - : +- ^ ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner (92) - : :- ^ InputIteratorTransformer (83) - : : +- ^ InputAdapter (82) - : : +- ^ ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ^ InputAdapter (67) - : : : +- ^ ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ^ InputAdapter (52) - : : : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ^ InputAdapter (37) - : : : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ^ InputAdapter (22) - : : : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ^ InputAdapter (7) - : : : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ^ InputAdapter (15) - : : : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ^ InputAdapter (30) - : : : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ^ InputAdapter (45) - : : : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ^ InputAdapter (60) - : : : +- ^ ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ^ InputAdapter (75) - : : +- ^ ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ^ InputAdapter (90) - : +- ^ ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ^ InputAdapter (105) - +- ^ ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (117) + +- ^ SortExecTransformer (115) + +- ^ InputIteratorTransformer (114) + +- ^ InputAdapter (113) + +- ^ ShuffleQueryStage (112), Statistics(X) + +- ColumnarExchange (111) + +- ^ ProjectExecTransformer (109) + +- ^ RegularHashAggregateExecTransformer (108) + +- ^ InputIteratorTransformer (107) + +- ^ InputAdapter (106) + +- ^ ShuffleQueryStage (105), Statistics(X) + +- ColumnarExchange (104) + +- ^ ProjectExecTransformer (102) + +- ^ FlushableHashAggregateExecTransformer (101) + +- ^ ProjectExecTransformer (100) + +- ^ ShuffledHashJoinExecTransformer Inner (99) + :- ^ InputIteratorTransformer (91) + : +- ^ InputAdapter (90) + : +- ^ ShuffleQueryStage (89), Statistics(X) + : +- ColumnarExchange (88) + : +- ^ ProjectExecTransformer (86) + : +- ^ ShuffledHashJoinExecTransformer Inner (85) + : :- ^ InputIteratorTransformer (77) + : : +- ^ InputAdapter (76) + : : +- ^ ShuffleQueryStage (75), Statistics(X) + : : +- ColumnarExchange (74) + : : +- ^ ProjectExecTransformer (72) + : : +- ^ ShuffledHashJoinExecTransformer Inner (71) + : : :- ^ InputIteratorTransformer (63) + : : : +- ^ InputAdapter (62) + : : : +- ^ ShuffleQueryStage (61), Statistics(X) + : : : +- ColumnarExchange (60) + : : : +- ^ ProjectExecTransformer (58) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (57) + : : : :- ^ InputIteratorTransformer (49) + : : : : +- ^ InputAdapter (48) + : : : : +- ^ ShuffleQueryStage (47), Statistics(X) + : : : : +- ColumnarExchange (46) + : : : : +- ^ ProjectExecTransformer (44) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (43) + : : : : :- ^ InputIteratorTransformer (35) + : : : : : +- ^ InputAdapter (34) + : : : : : +- ^ ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- ^ ProjectExecTransformer (30) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : : : : : :- ^ InputIteratorTransformer (21) + : : : : : : +- ^ InputAdapter (20) + : : : : : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : : : : : +- ColumnarExchange (18) + : : : : : : +- ^ ProjectExecTransformer (16) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : : : : : :- ^ InputIteratorTransformer (7) + : : : : : : : +- ^ InputAdapter (6) + : : : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : : : : : : +- ColumnarExchange (4) + : : : : : : : +- ^ ProjectExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (14) + : : : : : : +- ^ InputAdapter (13) + : : : : : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : : : : : +- ColumnarExchange (11) + : : : : : : +- ^ ProjectExecTransformer (9) + : : : : : : +- ^ Scan parquet (8) + : : : : : +- ^ InputIteratorTransformer (28) + : : : : : +- ^ InputAdapter (27) + : : : : : +- ^ ShuffleQueryStage (26), Statistics(X) + : : : : : +- ColumnarExchange (25) + : : : : : +- ^ ProjectExecTransformer (23) + : : : : : +- ^ Scan parquet (22) + : : : : +- ^ InputIteratorTransformer (42) + : : : : +- ^ InputAdapter (41) + : : : : +- ^ ShuffleQueryStage (40), Statistics(X) + : : : : +- ColumnarExchange (39) + : : : : +- ^ ProjectExecTransformer (37) + : : : : +- ^ Scan parquet (36) + : : : +- ^ InputIteratorTransformer (56) + : : : +- ^ InputAdapter (55) + : : : +- ^ ShuffleQueryStage (54), Statistics(X) + : : : +- ColumnarExchange (53) + : : : +- ^ ProjectExecTransformer (51) + : : : +- ^ Scan parquet (50) + : : +- ^ InputIteratorTransformer (70) + : : +- ^ InputAdapter (69) + : : +- ^ ShuffleQueryStage (68), Statistics(X) + : : +- ColumnarExchange (67) + : : +- ^ ProjectExecTransformer (65) + : : +- ^ Scan parquet (64) + : +- ^ InputIteratorTransformer (84) + : +- ^ InputAdapter (83) + : +- ^ ShuffleQueryStage (82), Statistics(X) + : +- ColumnarExchange (81) + : +- ^ ProjectExecTransformer (79) + : +- ^ Scan parquet (78) + +- ^ InputIteratorTransformer (98) + +- ^ InputAdapter (97) + +- ^ ShuffleQueryStage (96), Statistics(X) + +- ColumnarExchange (95) + +- ^ ProjectExecTransformer (93) + +- ^ Scan parquet (92) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (168) + +- Exchange (167) + +- HashAggregate (166) + +- Exchange (165) + +- HashAggregate (164) + +- Project (163) + +- ShuffledHashJoin Inner BuildRight (162) + :- Exchange (157) + : +- Project (156) + : +- ShuffledHashJoin Inner BuildRight (155) + : :- Exchange (151) + : : +- Project (150) + : : +- ShuffledHashJoin Inner BuildRight (149) + : : :- Exchange (145) + : : : +- Project (144) + : : : +- ShuffledHashJoin Inner BuildRight (143) + : : : :- Exchange (139) + : : : : +- Project (138) + : : : : +- ShuffledHashJoin Inner BuildRight (137) + : : : : :- Exchange (133) + : : : : : +- Project (132) + : : : : : +- ShuffledHashJoin Inner BuildRight (131) + : : : : : :- Exchange (127) + : : : : : : +- Project (126) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (125) + : : : : : : :- Exchange (121) + : : : : : : : +- Project (120) + : : : : : : : +- Filter (119) + : : : : : : : +- Scan parquet (118) + : : : : : : +- Exchange (124) + : : : : : : +- Filter (123) + : : : : : : +- Scan parquet (122) + : : : : : +- Exchange (130) + : : : : : +- Filter (129) + : : : : : +- Scan parquet (128) + : : : : +- Exchange (136) + : : : : +- Filter (135) + : : : : +- Scan parquet (134) + : : : +- Exchange (142) + : : : +- Filter (141) + : : : +- Scan parquet (140) + : : +- Exchange (148) + : : +- Filter (147) + : : +- Scan parquet (146) + : +- Exchange (154) + : +- Filter (153) + : +- Scan parquet (152) + +- Exchange (161) + +- Project (160) + +- Filter (159) + +- Scan parquet (158) (1) Scan parquet @@ -170,758 +162,726 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_type#X] -Arguments: ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(8) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(22) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(36) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(46) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(47) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(48) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(49) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(50) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) FilterExecTransformer -Input [2]: [c_custkey#X, c_nationkey#X] -Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) - -(56) ProjectExecTransformer +(51) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(53) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(55) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(60) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(61) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(62) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(63) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(64) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) FilterExecTransformer -Input [2]: [n_nationkey#X, n_regionkey#X] -Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) - -(71) ProjectExecTransformer +(65) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(67) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(69) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(70) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(71) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(72) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(86) ProjectExecTransformer +(79) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(81) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(82) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(83) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(84) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(85) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) ProjectExecTransformer +(86) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(88) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(89) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(90) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(91) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(92) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) FilterExecTransformer -Input [2]: [r_regionkey#X, r_name#X] -Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) - -(101) ProjectExecTransformer +(93) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(94) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(95) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(96) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(97) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(98) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(99) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(108) ProjectExecTransformer +(100) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(101) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(102) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(103) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(104) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(105) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(106) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(107) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(108) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(109) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(110) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(111) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(112) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(113) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(114) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(115) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(116) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(117) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(118) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(119) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(120) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(121) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(122) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(123) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(124) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(134) Project +(126) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(127) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(128) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(129) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(130) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(140) Project +(132) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(133) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(134) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(135) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(136) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(146) Project +(138) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(139) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(140) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(141) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(142) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(143) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(152) Project +(144) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(145) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(146) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(147) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(148) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(149) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(158) Project +(150) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(151) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(152) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(153) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(154) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(155) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(164) Project +(156) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(157) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(158) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(159) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(160) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(161) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(162) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(171) Project +(163) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(164) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(165) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(166) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(175) Exchange +(167) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(168) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(169) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt index aef854c3549e..0d3c59505d07 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt @@ -1,126 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ^ InputAdapter (90) - +- ^ ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ^ InputAdapter (84) - +- ^ ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner (77) - :- ^ InputIteratorTransformer (68) - : +- ^ InputAdapter (67) - : +- ^ ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner (62) - : :- ^ InputIteratorTransformer (53) - : : +- ^ InputAdapter (52) - : : +- ^ ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ^ InputAdapter (37) - : : : +- ^ ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ^ InputAdapter (22) - : : : : +- ^ ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ^ InputAdapter (7) - : : : : : +- ^ ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ^ InputAdapter (15) - : : : : +- ^ ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ^ InputAdapter (30) - : : : +- ^ ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ^ InputAdapter (45) - : : +- ^ ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ^ InputAdapter (60) - : +- ^ ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ^ InputAdapter (75) - +- ^ ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (88) + +- ^ SortExecTransformer (86) + +- ^ InputIteratorTransformer (85) + +- ^ InputAdapter (84) + +- ^ ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ^ InputAdapter (78) + +- ^ ShuffleQueryStage (77), Statistics(X) + +- ColumnarExchange (76) + +- ^ ProjectExecTransformer (74) + +- ^ FlushableHashAggregateExecTransformer (73) + +- ^ ProjectExecTransformer (72) + +- ^ ShuffledHashJoinExecTransformer Inner (71) + :- ^ InputIteratorTransformer (63) + : +- ^ InputAdapter (62) + : +- ^ ShuffleQueryStage (61), Statistics(X) + : +- ColumnarExchange (60) + : +- ^ ProjectExecTransformer (58) + : +- ^ ShuffledHashJoinExecTransformer Inner (57) + : :- ^ InputIteratorTransformer (49) + : : +- ^ InputAdapter (48) + : : +- ^ ShuffleQueryStage (47), Statistics(X) + : : +- ColumnarExchange (46) + : : +- ^ ProjectExecTransformer (44) + : : +- ^ ShuffledHashJoinExecTransformer Inner (43) + : : :- ^ InputIteratorTransformer (35) + : : : +- ^ InputAdapter (34) + : : : +- ^ ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- ^ ProjectExecTransformer (30) + : : : +- ^ ShuffledHashJoinExecTransformer Inner (29) + : : : :- ^ InputIteratorTransformer (21) + : : : : +- ^ InputAdapter (20) + : : : : +- ^ ShuffleQueryStage (19), Statistics(X) + : : : : +- ColumnarExchange (18) + : : : : +- ^ ProjectExecTransformer (16) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner (15) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- ^ InputAdapter (6) + : : : : : +- ^ ShuffleQueryStage (5), Statistics(X) + : : : : : +- ColumnarExchange (4) + : : : : : +- ^ ProjectExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (14) + : : : : +- ^ InputAdapter (13) + : : : : +- ^ ShuffleQueryStage (12), Statistics(X) + : : : : +- ColumnarExchange (11) + : : : : +- ^ ProjectExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (28) + : : : +- ^ InputAdapter (27) + : : : +- ^ ShuffleQueryStage (26), Statistics(X) + : : : +- ColumnarExchange (25) + : : : +- ^ ProjectExecTransformer (23) + : : : +- ^ Scan parquet (22) + : : +- ^ InputIteratorTransformer (42) + : : +- ^ InputAdapter (41) + : : +- ^ ShuffleQueryStage (40), Statistics(X) + : : +- ColumnarExchange (39) + : : +- ^ ProjectExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (56) + : +- ^ InputAdapter (55) + : +- ^ ShuffleQueryStage (54), Statistics(X) + : +- ColumnarExchange (53) + : +- ^ ProjectExecTransformer (51) + : +- ^ Scan parquet (50) + +- ^ InputIteratorTransformer (70) + +- ^ InputAdapter (69) + +- ^ ShuffleQueryStage (68), Statistics(X) + +- ColumnarExchange (67) + +- ^ ProjectExecTransformer (65) + +- ^ Scan parquet (64) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (126) + +- Exchange (125) + +- HashAggregate (124) + +- Exchange (123) + +- HashAggregate (122) + +- Project (121) + +- ShuffledHashJoin Inner BuildRight (120) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildRight (108) + : : :- Exchange (104) + : : : +- Project (103) + : : : +- ShuffledHashJoin Inner BuildRight (102) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- ShuffledHashJoin Inner BuildLeft (96) + : : : : :- Exchange (92) + : : : : : +- Project (91) + : : : : : +- Filter (90) + : : : : : +- Scan parquet (89) + : : : : +- Exchange (95) + : : : : +- Filter (94) + : : : : +- Scan parquet (93) + : : : +- Exchange (101) + : : : +- Filter (100) + : : : +- Scan parquet (99) + : : +- Exchange (107) + : : +- Filter (106) + : : +- Scan parquet (105) + : +- Exchange (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -130,570 +124,546 @@ Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(2) FilterExecTransformer -Input [2]: [p_partkey#X, p_name#X] -Arguments: ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) - -(3) ProjectExecTransformer +(2) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(4) WholeStageCodegenTransformer (X) +(3) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(4) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(5) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(6) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(7) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(8) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer -Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) - -(11) ProjectExecTransformer +(9) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(10) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(11) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(12) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(13) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(14) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(15) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(16) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(17) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(18) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(19) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(20) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(21) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(22) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer -Input [2]: [s_suppkey#X, s_nationkey#X] -Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) - -(26) ProjectExecTransformer +(23) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(25) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(26) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(27) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(28) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(29) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(30) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(32) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(33) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(34) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(35) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(36) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) FilterExecTransformer -Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) - -(41) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(39) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(40) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(41) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(42) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(43) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(44) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(45) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(46) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(47) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(48) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(49) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(50) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) FilterExecTransformer -Input [2]: [o_orderkey#X, o_orderdate#X] -Arguments: isnotnull(o_orderkey#X) - -(56) ProjectExecTransformer +(51) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(52) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(53) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(55) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(57) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(58) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(60) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(61) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(62) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(63) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(64) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer -Input [2]: [n_nationkey#X, n_name#X] -Arguments: isnotnull(n_nationkey#X) - -(71) ProjectExecTransformer +(65) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(66) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(67) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(68) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(69) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(70) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(71) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(72) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(73) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(74) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(76) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(77) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(78) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(79) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(82) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(83) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(84) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(85) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(86) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(87) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(88) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(89) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(90) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(91) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(92) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(93) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(94) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(95) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(103) Project +(97) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(98) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(99) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(100) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(101) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(102) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(109) Project +(103) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(104) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(105) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(106) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(107) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(115) Project +(109) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(110) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(113) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(121) Project +(115) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(116) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(117) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(118) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(119) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(120) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(127) Project +(121) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(122) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(123) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(124) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(125) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(126) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala index fe2b74012929..e703295d0851 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicPhysicalOperatorTransformer.scala @@ -152,6 +152,14 @@ abstract class FilterExecTransformerBase(val cond: Expression, val input: SparkP } } +object FilterExecTransformerBase { + implicit class FilterExecTransformerBaseImplicits(filter: FilterExecTransformerBase) { + def isNoop(): Boolean = { + filter.getRemainingCondition == null + } + } +} + case class ProjectExecTransformer private (projectList: Seq[NamedExpression], child: SparkPlan) extends UnaryTransformSupport with OrderPreservingNodeShim @@ -361,12 +369,12 @@ object FilterHandler extends PredicateHelper { // Separate and compare the filter conditions in Scan and Filter. // Try to push down the remaining conditions in Filter into Scan. - def applyFilterPushdownToScan(filter: FilterExec): SparkPlan = - filter.child match { + def pushFilterToScan(condition: Expression, scan: SparkPlan): SparkPlan = + scan match { case fileSourceScan: FileSourceScanExec => val pushDownFilters = BackendsApiManager.getSparkPlanExecApiInstance.postProcessPushDownFilter( - splitConjunctivePredicates(filter.condition), + splitConjunctivePredicates(condition), fileSourceScan) ScanTransformerFactory.createFileSourceScanTransformer( fileSourceScan, @@ -374,7 +382,7 @@ object FilterHandler extends PredicateHelper { case batchScan: BatchScanExec => val pushDownFilters = BackendsApiManager.getSparkPlanExecApiInstance.postProcessPushDownFilter( - splitConjunctivePredicates(filter.condition), + splitConjunctivePredicates(condition), batchScan) ScanTransformerFactory.createBatchScanTransformer( batchScan, diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala index 0fd0856f2cda..02a466b6ae2a 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/MiscColumnarRules.scala @@ -30,12 +30,12 @@ object MiscColumnarRules { object TransformPreOverrides { def apply(): TransformPreOverrides = { TransformPreOverrides( - List(ImplementFilter()), + List(TransformFilter()), List( - ImplementOthers(), - ImplementAggregate(), - ImplementExchange(), - ImplementJoin() + TransformOthers(), + TransformAggregate(), + TransformExchange(), + TransformJoin() ) ) } @@ -43,8 +43,8 @@ object MiscColumnarRules { // This rule will conduct the conversion from Spark plan to the plan transformer. case class TransformPreOverrides( - topDownRules: Seq[ImplementSingleNode], - bottomUpRules: Seq[ImplementSingleNode]) + topDownRules: Seq[TransformSingleNode], + bottomUpRules: Seq[TransformSingleNode]) extends Rule[SparkPlan] with LogLevelUtil { @transient private val planChangeLogger = new PlanChangeLogger[SparkPlan]() diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ImplementSingleNode.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformSingleNode.scala similarity index 95% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ImplementSingleNode.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformSingleNode.scala index 4b810af14a35..d005baa30605 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/ImplementSingleNode.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/TransformSingleNode.scala @@ -40,12 +40,12 @@ import org.apache.spark.sql.execution.python.EvalPythonExec import org.apache.spark.sql.execution.window.WindowExec import org.apache.spark.sql.hive.HiveTableScanExecTransformer -sealed trait ImplementSingleNode extends Logging { +sealed trait TransformSingleNode extends Logging { def impl(plan: SparkPlan): SparkPlan } // Aggregation transformation. -case class ImplementAggregate() extends ImplementSingleNode with LogLevelUtil { +case class TransformAggregate() extends TransformSingleNode with LogLevelUtil { override def impl(plan: SparkPlan): SparkPlan = plan match { case plan if TransformHints.isNotTransformable(plan) => plan @@ -107,7 +107,7 @@ case class ImplementAggregate() extends ImplementSingleNode with LogLevelUtil { } // Exchange transformation. -case class ImplementExchange() extends ImplementSingleNode with LogLevelUtil { +case class TransformExchange() extends TransformSingleNode with LogLevelUtil { override def impl(plan: SparkPlan): SparkPlan = plan match { case plan if TransformHints.isNotTransformable(plan) => plan @@ -131,7 +131,7 @@ case class ImplementExchange() extends ImplementSingleNode with LogLevelUtil { } // Join transformation. -case class ImplementJoin() extends ImplementSingleNode with LogLevelUtil { +case class TransformJoin() extends TransformSingleNode with LogLevelUtil { /** * Get the build side supported by the execution of vanilla Spark. @@ -238,8 +238,8 @@ case class ImplementJoin() extends ImplementSingleNode with LogLevelUtil { } // Filter transformation. -case class ImplementFilter() extends ImplementSingleNode with LogLevelUtil { - import ImplementOthers._ +case class TransformFilter() extends TransformSingleNode with LogLevelUtil { + import TransformOthers._ private val replace = new ReplaceSingleNode() override def impl(plan: SparkPlan): SparkPlan = plan match { @@ -251,24 +251,24 @@ case class ImplementFilter() extends ImplementSingleNode with LogLevelUtil { /** * Generate a plan for filter. * - * @param plan + * @param filter * : the original Spark plan. * @return * the actually used plan for execution. */ - private def genFilterExec(plan: FilterExec): SparkPlan = { - if (TransformHints.isNotTransformable(plan)) { - return plan + private def genFilterExec(filter: FilterExec): SparkPlan = { + if (TransformHints.isNotTransformable(filter)) { + return filter } // FIXME: Filter push-down should be better done by Vanilla Spark's planner or by // a individual rule. - val scan = plan.child // Push down the left conditions in Filter into FileSourceScan. - val newChild: SparkPlan = scan match { - case _: FileSourceScanExec | _: BatchScanExec => + val newChild: SparkPlan = filter.child match { + case scan @ (_: FileSourceScanExec | _: BatchScanExec) => if (TransformHints.isTransformable(scan)) { - val newScan = FilterHandler.applyFilterPushdownToScan(plan) + val newScan = + FilterHandler.pushFilterToScan(filter.condition, scan) newScan match { case ts: TransformSupport if ts.doValidate().isValid => ts // TODO remove the call @@ -277,23 +277,23 @@ case class ImplementFilter() extends ImplementSingleNode with LogLevelUtil { } else { replace.doReplace(scan) } - case _ => replace.doReplace(plan.child) + case _ => replace.doReplace(filter.child) } - logDebug(s"Columnar Processing for ${plan.getClass} is currently supported.") + logDebug(s"Columnar Processing for ${filter.getClass} is currently supported.") BackendsApiManager.getSparkPlanExecApiInstance - .genFilterExecTransformer(plan.condition, newChild) + .genFilterExecTransformer(filter.condition, newChild) } } // Other transformations. -case class ImplementOthers() extends ImplementSingleNode with LogLevelUtil { - import ImplementOthers._ +case class TransformOthers() extends TransformSingleNode with LogLevelUtil { + import TransformOthers._ private val replace = new ReplaceSingleNode() override def impl(plan: SparkPlan): SparkPlan = replace.doReplace(plan) } -object ImplementOthers { +object TransformOthers { // Utility to replace single node within transformed Gluten node. // Children will be preserved as they are as children of the output node. // diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala index c2596966759c..b7879b900531 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedApplier.scala @@ -19,7 +19,7 @@ package org.apache.gluten.extension.columnar.enumerated import org.apache.gluten.GlutenConfig import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.extension.columnar._ -import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow, TransformPostOverrides, TransformPreOverrides} +import org.apache.gluten.extension.columnar.MiscColumnarRules.{RemoveGlutenTableCacheColumnarToRow, RemoveTopmostColumnarToRow, TransformPostOverrides} import org.apache.gluten.extension.columnar.util.AdaptiveContext import org.apache.gluten.metrics.GlutenTimeMetric import org.apache.gluten.utils.{LogLevelUtil, PhysicalPlanSelector} @@ -125,7 +125,6 @@ class EnumeratedApplier(session: SparkSession) (_: SparkSession) => FallbackBloomFilterAggIfNeeded() ) ::: List( - (_: SparkSession) => TransformPreOverrides(List(ImplementFilter()), List.empty), (session: SparkSession) => EnumeratedTransform(session, outputsColumnar), (_: SparkSession) => RemoveTransitions ) ::: diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala index 471047fac7a1..4ce768a06248 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/EnumeratedTransform.scala @@ -16,7 +16,7 @@ */ package org.apache.gluten.extension.columnar.enumerated -import org.apache.gluten.extension.columnar.{ImplementExchange, ImplementJoin, ImplementOthers, ImplementSingleNode} +import org.apache.gluten.extension.columnar.{TransformExchange, TransformJoin, TransformOthers, TransformSingleNode} import org.apache.gluten.planner.GlutenOptimization import org.apache.gluten.planner.property.Conventions import org.apache.gluten.ras.property.PropertySet @@ -33,10 +33,13 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean) import EnumeratedTransform._ private val rasRules = List( - AsRasImplement(ImplementOthers()), - AsRasImplement(ImplementExchange()), - AsRasImplement(ImplementJoin()), - RasImplementAggregate + AsRasImplement(TransformOthers()), + AsRasImplement(TransformExchange()), + AsRasImplement(TransformJoin()), + ImplementAggregate, + ImplementFilter, + PushFilterToScan, + FilterRemoveRule ) private val optimization = GlutenOptimization(rasRules) @@ -56,7 +59,7 @@ case class EnumeratedTransform(session: SparkSession, outputsColumnar: Boolean) } object EnumeratedTransform { - private case class AsRasImplement(delegate: ImplementSingleNode) extends RasRule[SparkPlan] { + private case class AsRasImplement(delegate: TransformSingleNode) extends RasRule[SparkPlan] { override def shift(node: SparkPlan): Iterable[SparkPlan] = { val out = List(delegate.impl(node)) out diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/FilterRemoveRule.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/FilterRemoveRule.scala new file mode 100644 index 000000000000..52b5be981194 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/FilterRemoveRule.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.enumerated + +import org.apache.gluten.execution.{BasicScanExecTransformer, FilterExecTransformerBase} +import org.apache.gluten.ras.path.Pattern._ +import org.apache.gluten.ras.path.Pattern.Matchers._ +import org.apache.gluten.ras.rule.{RasRule, Shape} +import org.apache.gluten.ras.rule.Shapes._ + +import org.apache.spark.sql.execution.SparkPlan + +// Removes Gluten filter operator if its no-op. Typically a Gluten filter is no-op when it +// pushes all of its conditions into the child scan. +// +// The rule is needed in RAS since our cost model treats all filter + scan plans with constant cost +// because the pushed filter is not considered in the model. Removing the filter will make +// optimizer choose a single scan as the winner sub-plan since a single scan's cost is lower than +// filter + scan. +object FilterRemoveRule extends RasRule[SparkPlan] { + override def shift(node: SparkPlan): Iterable[SparkPlan] = { + val filter = node.asInstanceOf[FilterExecTransformerBase] + if (filter.isNoop()) { + return List(filter.child) + } + List.empty + } + + override def shape(): Shape[SparkPlan] = + pattern( + node[SparkPlan]( + clazz(classOf[FilterExecTransformerBase]), + leaf(clazz(classOf[BasicScanExecTransformer])) + ).build()) +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasImplementAggregate.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ImplementAggregate.scala similarity index 97% rename from gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasImplementAggregate.scala rename to gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ImplementAggregate.scala index deb80dac3bc4..818d225684f6 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/RasImplementAggregate.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ImplementAggregate.scala @@ -23,7 +23,7 @@ import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes} import org.apache.spark.sql.execution.SparkPlan import org.apache.spark.sql.execution.aggregate.HashAggregateExec -object RasImplementAggregate extends RasRule[SparkPlan] { +object ImplementAggregate extends RasRule[SparkPlan] { override def shift(node: SparkPlan): Iterable[SparkPlan] = node match { case plan if TransformHints.isNotTransformable(plan) => List.empty case agg: HashAggregateExec => shiftAgg(agg) diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ImplementFilter.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ImplementFilter.scala new file mode 100644 index 000000000000..6ec384bd3e1a --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/ImplementFilter.scala @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.enumerated + +import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.extension.columnar.TransformHints +import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes} + +import org.apache.spark.sql.execution.{FilterExec, SparkPlan} + +object ImplementFilter extends RasRule[SparkPlan] { + override def shift(node: SparkPlan): Iterable[SparkPlan] = node match { + case plan if TransformHints.isNotTransformable(plan) => List.empty + case FilterExec(condition, child) => + List( + BackendsApiManager.getSparkPlanExecApiInstance + .genFilterExecTransformer(condition, child)) + case _ => + List.empty + } + override def shape(): Shape[SparkPlan] = Shapes.fixedHeight(1) +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala new file mode 100644 index 000000000000..f04f572c1ac9 --- /dev/null +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/columnar/enumerated/PushFilterToScan.scala @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gluten.extension.columnar.enumerated + +import org.apache.gluten.execution.{FilterHandler, TransformSupport} +import org.apache.gluten.extension.columnar.TransformHints +import org.apache.gluten.ras.path.Pattern._ +import org.apache.gluten.ras.path.Pattern.Matchers._ +import org.apache.gluten.ras.rule.{RasRule, Shape} +import org.apache.gluten.ras.rule.Shapes._ + +import org.apache.spark.sql.execution.{ColumnarToRowExec, ColumnarToRowTransition, FileSourceScanExec, FilterExec, SparkPlan} +import org.apache.spark.sql.execution.datasources.v2.BatchScanExec + +object PushFilterToScan extends RasRule[SparkPlan] { + override def shift(node: SparkPlan): Iterable[SparkPlan] = node match { + case FilterAndScan(filter, scan) => + if (!TransformHints.isTransformable(scan)) { + return List.empty + } + val newScan = + FilterHandler.pushFilterToScan(filter.condition, scan) + newScan match { + case ts: TransformSupport if ts.doValidate().isValid => + List(filter.withNewChildren(List(ts))) + case _ => + List.empty + } + case _ => + List.empty + } + + override def shape(): Shape[SparkPlan] = + anyOf( + pattern( + node[SparkPlan]( + clazz(classOf[FilterExec]), + leaf( + or(clazz(classOf[FileSourceScanExec]), clazz(classOf[BatchScanExec])) + ) + ).build()), + pattern( + node[SparkPlan]( + clazz(classOf[FilterExec]), + node( + clazz(classOf[ColumnarToRowTransition]), + leaf( + or(clazz(classOf[FileSourceScanExec]), clazz(classOf[BatchScanExec])) + ) + ) + ).build()) + ) + + private object FilterAndScan { + def unapply(node: SparkPlan): Option[(FilterExec, SparkPlan)] = node match { + case f @ FilterExec(cond, ColumnarToRowExec(scan)) => + Some(f, scan) + case f @ FilterExec(cond, scan) => + Some(f, scan) + case _ => + None + } + } +} diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/GlutenMetadata.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/GlutenMetadata.scala index 7eb1d4b57e42..e25f0a1f1c06 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/GlutenMetadata.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/metadata/GlutenMetadata.scala @@ -30,6 +30,30 @@ object GlutenMetadata { Impl(schema) } - private case class Impl(schema: Schema) extends GlutenMetadata - case class Schema(output: Seq[Attribute]) + private case class Impl(override val schema: Schema) extends GlutenMetadata + + case class Schema(output: Seq[Attribute]) { + private val hash = output.map(_.semanticHash()).hashCode() + + override def hashCode(): Int = { + hash + } + + override def equals(obj: Any): Boolean = obj match { + case other: Schema => + semanticEquals(other) + case _ => + false + } + + private def semanticEquals(other: Schema): Boolean = { + if (output.size != other.output.size) { + return false + } + output.zip(other.output).forall { + case (left, right) => + left.semanticEquals(right) + } + } + } } diff --git a/gluten-core/src/main/scala/org/apache/gluten/planner/property/Convention.scala b/gluten-core/src/main/scala/org/apache/gluten/planner/property/Convention.scala index 577cf43ba278..36751f4ca188 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/planner/property/Convention.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/planner/property/Convention.scala @@ -17,6 +17,7 @@ package org.apache.gluten.planner.property import org.apache.gluten.backendsapi.BackendsApiManager +import org.apache.gluten.extension.GlutenPlan import org.apache.gluten.extension.columnar.ColumnarTransitions import org.apache.gluten.planner.plan.GlutenPlanModel.GroupLeafExec import org.apache.gluten.ras.{Property, PropertyDef} @@ -24,8 +25,9 @@ import org.apache.gluten.ras.rule.{RasRule, Shape, Shapes} import org.apache.gluten.sql.shims.SparkShimLoader import org.apache.gluten.utils.PlanUtil -import org.apache.spark.sql.execution.{ColumnarToRowExec, InputAdapter, RowToColumnarExec, SparkPlan, WholeStageCodegenExec} -import org.apache.spark.sql.execution.adaptive.{AQEShuffleReadExec, QueryStageExec} +import org.apache.spark.sql.execution._ +import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, AQEShuffleReadExec, QueryStageExec} +import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec import org.apache.spark.sql.execution.exchange.ReusedExchangeExec sealed trait Convention extends Property[SparkPlan] { @@ -53,14 +55,27 @@ object ConventionDef extends PropertyDef[SparkPlan, Convention] { // convention-propagation. Probably need to refactor getChildrenPropertyRequirements. override def getProperty(plan: SparkPlan): Convention = plan match { case _: GroupLeafExec => throw new IllegalStateException() + case other => conventionOf(other) + } + + private def conventionOf(plan: SparkPlan): Convention = plan match { + case g: GroupLeafExec => g.propertySet.get(ConventionDef) case ColumnarToRowExec(child) => Conventions.ROW_BASED case RowToColumnarExec(child) => Conventions.VANILLA_COLUMNAR case ColumnarTransitions.ColumnarToRowLike(child) => Conventions.ROW_BASED case ColumnarTransitions.RowToColumnarLike(child) => Conventions.GLUTEN_COLUMNAR - case p if PlanUtil.outputNativeColumnarData(p) => Conventions.GLUTEN_COLUMNAR - case p if PlanUtil.isVanillaColumnarOp(p) => Conventions.VANILLA_COLUMNAR + case q: QueryStageExec => conventionOf(q.plan) + case r: ReusedExchangeExec => conventionOf(r.child) + case a: AdaptiveSparkPlanExec => conventionOf(a.executedPlan) + case i: InMemoryTableScanExec => getCacheConvention(i) + case p if canPropagateConvention(p) => + val childrenProps = p.children.map(conventionOf).distinct + assert(childrenProps.size == 1) + childrenProps.head + case _: GlutenPlan => Conventions.GLUTEN_COLUMNAR + case p if p.supportsColumnar => Conventions.VANILLA_COLUMNAR case p if SparkShimLoader.getSparkShims.supportsRowBased(p) => Conventions.ROW_BASED - case _ => throw new IllegalStateException() + case other => throw new IllegalStateException(s"Unable to get convention of $other") } override def getChildrenConstraints( @@ -70,22 +85,30 @@ object ConventionDef extends PropertyDef[SparkPlan, Convention] { case ColumnarTransitions.ColumnarToRowLike(child) => Seq(Conventions.GLUTEN_COLUMNAR) case ColumnarTransitions.RowToColumnarLike(child) => Seq(Conventions.ROW_BASED) case p if canPropagateConvention(p) => - plan.children.map(_ => constraint.asInstanceOf[Convention]) - case _ => - val conv = getProperty(plan) - plan.children.map(_ => conv) + p.children.map(_ => constraint.asInstanceOf[Convention]) + case other => + val conv = conventionOf(other) + other.children.map(_ => conv) } override def any(): Convention = Conventions.ANY private def canPropagateConvention(plan: SparkPlan): Boolean = plan match { case p: AQEShuffleReadExec => true - case p: QueryStageExec => true - case p: ReusedExchangeExec => true case p: InputAdapter => true case p: WholeStageCodegenExec => true case _ => false } + + private def getCacheConvention(i: InMemoryTableScanExec): Convention = { + if (PlanUtil.isGlutenTableCache(i)) { + Conventions.GLUTEN_COLUMNAR + } else if (i.supportsColumnar) { + Conventions.VANILLA_COLUMNAR + } else { + Conventions.ROW_BASED + } + } } case class ConventionEnforcerRule(reqConv: Convention) extends RasRule[SparkPlan] { diff --git a/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala b/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala index ad9d95daec0e..15aa11a13fd8 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/utils/PlanUtil.scala @@ -24,7 +24,7 @@ import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec import org.apache.spark.sql.execution.exchange._ object PlanUtil { - private def isGlutenTableCacheInternal(i: InMemoryTableScanExec): Boolean = { + def isGlutenTableCacheInternal(i: InMemoryTableScanExec): Boolean = { // `ColumnarCachedBatchSerializer` is at velox module, so use class name here i.relation.cacheBuilder.serializer.getClass.getSimpleName == "ColumnarCachedBatchSerializer" && i.supportsColumnar diff --git a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/rule/EnforcerRule.scala b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/rule/EnforcerRule.scala index 439b88a2cb57..c82deb71382f 100644 --- a/gluten-ras/common/src/main/scala/org/apache/gluten/ras/rule/EnforcerRule.scala +++ b/gluten-ras/common/src/main/scala/org/apache/gluten/ras/rule/EnforcerRule.scala @@ -45,6 +45,11 @@ object EnforcerRule { override def shape(): Shape[T] = rule.shape() } + // A built-in enforcer rule that does constraint propagation. The rule directly outputs + // whatever passed in, and memo will copy the output node in with the desired constraint. + // During witch children constraints will be derived through PropertyDef#getChildrenConstraints. + // When the children constraints do changed, the new node with changed children constraints will + // be persisted into memo. private class BuiltinEnforcerRule[T <: AnyRef](override val constraint: Property[T]) extends EnforcerRule[T] { override def shift(node: T): Iterable[T] = List(node)