From 36ecb49babc33c50ab20f100d2ae91f54faaf383 Mon Sep 17 00:00:00 2001 From: Hongze Zhang Date: Fri, 14 Jun 2024 14:04:17 +0800 Subject: [PATCH] fixup --- .../v1-bhj-ras/spark32/1.txt | 83 +- .../v1-bhj-ras/spark32/10.txt | 189 ++--- .../v1-bhj-ras/spark32/11.txt | 157 ++-- .../v1-bhj-ras/spark32/12.txt | 117 +-- .../v1-bhj-ras/spark32/13.txt | 137 ++-- .../v1-bhj-ras/spark32/14.txt | 105 +-- .../v1-bhj-ras/spark32/15.txt | 123 +-- .../v1-bhj-ras/spark32/16.txt | 155 ++-- .../v1-bhj-ras/spark32/17.txt | 127 ++-- .../v1-bhj-ras/spark32/18.txt | 306 ++++---- .../v1-bhj-ras/spark32/19.txt | 101 +-- .../v1-bhj-ras/spark32/20.txt | 398 +++++----- .../v1-bhj-ras/spark32/21.txt | 247 +++--- .../v1-bhj-ras/spark32/22.txt | 105 +-- .../v1-bhj-ras/spark32/3.txt | 155 ++-- .../v1-bhj-ras/spark32/4.txt | 123 +-- .../v1-bhj-ras/spark32/5.txt | 259 +++---- .../v1-bhj-ras/spark32/6.txt | 67 +- .../v1-bhj-ras/spark32/7.txt | 243 +++--- .../v1-bhj-ras/spark32/8.txt | 329 ++++---- .../v1-bhj-ras/spark32/9.txt | 253 ++++--- .../v1-bhj-ras/spark33/1.txt | 83 +- .../v1-bhj-ras/spark33/10.txt | 189 ++--- .../v1-bhj-ras/spark33/11.txt | 315 ++++---- .../v1-bhj-ras/spark33/12.txt | 117 +-- .../v1-bhj-ras/spark33/13.txt | 137 ++-- .../v1-bhj-ras/spark33/14.txt | 105 +-- .../v1-bhj-ras/spark33/15.txt | 209 +++--- .../v1-bhj-ras/spark33/16.txt | 155 ++-- .../v1-bhj-ras/spark33/17.txt | 127 ++-- .../v1-bhj-ras/spark33/18.txt | 306 ++++---- .../v1-bhj-ras/spark33/19.txt | 101 +-- .../v1-bhj-ras/spark33/20.txt | 390 +++++----- .../v1-bhj-ras/spark33/21.txt | 243 +++--- .../v1-bhj-ras/spark33/22.txt | 175 ++--- .../v1-bhj-ras/spark33/3.txt | 155 ++-- .../v1-bhj-ras/spark33/4.txt | 123 +-- .../v1-bhj-ras/spark33/5.txt | 259 +++---- .../v1-bhj-ras/spark33/6.txt | 67 +- .../v1-bhj-ras/spark33/7.txt | 243 +++--- .../v1-bhj-ras/spark33/8.txt | 329 ++++---- .../v1-bhj-ras/spark33/9.txt | 253 ++++--- .../v1-bhj-ras/spark34/1.txt | 83 +- .../v1-bhj-ras/spark34/10.txt | 189 ++--- .../v1-bhj-ras/spark34/11.txt | 315 ++++---- .../v1-bhj-ras/spark34/12.txt | 117 +-- .../v1-bhj-ras/spark34/13.txt | 137 ++-- .../v1-bhj-ras/spark34/14.txt | 105 +-- .../v1-bhj-ras/spark34/15.txt | 209 +++--- .../v1-bhj-ras/spark34/16.txt | 155 ++-- .../v1-bhj-ras/spark34/17.txt | 127 ++-- .../v1-bhj-ras/spark34/18.txt | 306 ++++---- .../v1-bhj-ras/spark34/19.txt | 101 +-- .../v1-bhj-ras/spark34/20.txt | 253 ++++--- .../v1-bhj-ras/spark34/21.txt | 243 +++--- .../v1-bhj-ras/spark34/22.txt | 175 ++--- .../v1-bhj-ras/spark34/3.txt | 155 ++-- .../v1-bhj-ras/spark34/4.txt | 123 +-- .../v1-bhj-ras/spark34/5.txt | 259 +++---- .../v1-bhj-ras/spark34/6.txt | 67 +- .../v1-bhj-ras/spark34/7.txt | 243 +++--- .../v1-bhj-ras/spark34/8.txt | 329 ++++---- .../v1-bhj-ras/spark34/9.txt | 253 ++++--- .../tpch-approved-plan/v1-bhj/spark32/1.txt | 83 +- .../tpch-approved-plan/v1-bhj/spark32/10.txt | 189 ++--- .../tpch-approved-plan/v1-bhj/spark32/11.txt | 157 ++-- .../tpch-approved-plan/v1-bhj/spark32/12.txt | 117 +-- .../tpch-approved-plan/v1-bhj/spark32/13.txt | 137 ++-- .../tpch-approved-plan/v1-bhj/spark32/14.txt | 105 +-- .../tpch-approved-plan/v1-bhj/spark32/15.txt | 123 +-- .../tpch-approved-plan/v1-bhj/spark32/16.txt | 155 ++-- .../tpch-approved-plan/v1-bhj/spark32/17.txt | 127 ++-- .../tpch-approved-plan/v1-bhj/spark32/18.txt | 306 ++++---- .../tpch-approved-plan/v1-bhj/spark32/19.txt | 101 +-- .../tpch-approved-plan/v1-bhj/spark32/20.txt | 398 +++++----- .../tpch-approved-plan/v1-bhj/spark32/21.txt | 247 +++--- .../tpch-approved-plan/v1-bhj/spark32/22.txt | 105 +-- .../tpch-approved-plan/v1-bhj/spark32/3.txt | 155 ++-- .../tpch-approved-plan/v1-bhj/spark32/4.txt | 123 +-- .../tpch-approved-plan/v1-bhj/spark32/5.txt | 259 +++---- .../tpch-approved-plan/v1-bhj/spark32/6.txt | 67 +- .../tpch-approved-plan/v1-bhj/spark32/7.txt | 243 +++--- .../tpch-approved-plan/v1-bhj/spark32/8.txt | 329 ++++---- .../tpch-approved-plan/v1-bhj/spark32/9.txt | 253 ++++--- .../tpch-approved-plan/v1-bhj/spark33/1.txt | 83 +- .../tpch-approved-plan/v1-bhj/spark33/10.txt | 189 ++--- .../tpch-approved-plan/v1-bhj/spark33/11.txt | 315 ++++---- .../tpch-approved-plan/v1-bhj/spark33/12.txt | 117 +-- .../tpch-approved-plan/v1-bhj/spark33/13.txt | 137 ++-- .../tpch-approved-plan/v1-bhj/spark33/14.txt | 105 +-- .../tpch-approved-plan/v1-bhj/spark33/15.txt | 209 +++--- .../tpch-approved-plan/v1-bhj/spark33/16.txt | 155 ++-- .../tpch-approved-plan/v1-bhj/spark33/17.txt | 127 ++-- .../tpch-approved-plan/v1-bhj/spark33/18.txt | 306 ++++---- .../tpch-approved-plan/v1-bhj/spark33/19.txt | 101 +-- .../tpch-approved-plan/v1-bhj/spark33/20.txt | 390 +++++----- .../tpch-approved-plan/v1-bhj/spark33/21.txt | 243 +++--- .../tpch-approved-plan/v1-bhj/spark33/22.txt | 207 ++--- .../tpch-approved-plan/v1-bhj/spark33/3.txt | 155 ++-- .../tpch-approved-plan/v1-bhj/spark33/4.txt | 123 +-- .../tpch-approved-plan/v1-bhj/spark33/5.txt | 259 +++---- .../tpch-approved-plan/v1-bhj/spark33/6.txt | 67 +- .../tpch-approved-plan/v1-bhj/spark33/7.txt | 243 +++--- .../tpch-approved-plan/v1-bhj/spark33/8.txt | 329 ++++---- .../tpch-approved-plan/v1-bhj/spark33/9.txt | 253 ++++--- .../tpch-approved-plan/v1-bhj/spark34/1.txt | 83 +- .../tpch-approved-plan/v1-bhj/spark34/10.txt | 189 ++--- .../tpch-approved-plan/v1-bhj/spark34/11.txt | 315 ++++---- .../tpch-approved-plan/v1-bhj/spark34/12.txt | 117 +-- .../tpch-approved-plan/v1-bhj/spark34/13.txt | 137 ++-- .../tpch-approved-plan/v1-bhj/spark34/14.txt | 105 +-- .../tpch-approved-plan/v1-bhj/spark34/15.txt | 209 +++--- .../tpch-approved-plan/v1-bhj/spark34/16.txt | 155 ++-- .../tpch-approved-plan/v1-bhj/spark34/17.txt | 127 ++-- .../tpch-approved-plan/v1-bhj/spark34/18.txt | 306 ++++---- .../tpch-approved-plan/v1-bhj/spark34/19.txt | 101 +-- .../tpch-approved-plan/v1-bhj/spark34/20.txt | 253 ++++--- .../tpch-approved-plan/v1-bhj/spark34/21.txt | 243 +++--- .../tpch-approved-plan/v1-bhj/spark34/22.txt | 207 ++--- .../tpch-approved-plan/v1-bhj/spark34/3.txt | 155 ++-- .../tpch-approved-plan/v1-bhj/spark34/4.txt | 123 +-- .../tpch-approved-plan/v1-bhj/spark34/5.txt | 259 +++---- .../tpch-approved-plan/v1-bhj/spark34/6.txt | 67 +- .../tpch-approved-plan/v1-bhj/spark34/7.txt | 243 +++--- .../tpch-approved-plan/v1-bhj/spark34/8.txt | 329 ++++---- .../tpch-approved-plan/v1-bhj/spark34/9.txt | 253 ++++--- .../tpch-approved-plan/v1-ras/spark32/1.txt | 83 +- .../tpch-approved-plan/v1-ras/spark32/10.txt | 340 +++++---- .../tpch-approved-plan/v1-ras/spark32/11.txt | 279 +++---- .../tpch-approved-plan/v1-ras/spark32/12.txt | 185 ++--- .../tpch-approved-plan/v1-ras/spark32/13.txt | 199 ++--- .../tpch-approved-plan/v1-ras/spark32/14.txt | 132 ++-- .../tpch-approved-plan/v1-ras/spark32/15.txt | 168 +++-- .../tpch-approved-plan/v1-ras/spark32/16.txt | 241 +++--- .../tpch-approved-plan/v1-ras/spark32/17.txt | 216 +++--- .../tpch-approved-plan/v1-ras/spark32/18.txt | 377 +++++----- .../tpch-approved-plan/v1-ras/spark32/19.txt | 128 ++-- .../tpch-approved-plan/v1-ras/spark32/20.txt | 502 +++++++------ .../tpch-approved-plan/v1-ras/spark32/21.txt | 471 ++++++------ .../tpch-approved-plan/v1-ras/spark32/22.txt | 173 +++-- .../tpch-approved-plan/v1-ras/spark32/3.txt | 230 +++--- .../tpch-approved-plan/v1-ras/spark32/4.txt | 189 ++--- .../tpch-approved-plan/v1-ras/spark32/5.txt | 533 +++++++------ .../tpch-approved-plan/v1-ras/spark32/6.txt | 67 +- .../tpch-approved-plan/v1-ras/spark32/7.txt | 506 +++++++------ .../tpch-approved-plan/v1-ras/spark32/8.txt | 709 ++++++++++-------- .../tpch-approved-plan/v1-ras/spark32/9.txt | 529 +++++++------ .../tpch-approved-plan/v1-ras/spark33/1.txt | 83 +- .../tpch-approved-plan/v1-ras/spark33/10.txt | 340 +++++---- .../tpch-approved-plan/v1-ras/spark33/11.txt | 473 ++++++------ .../tpch-approved-plan/v1-ras/spark33/12.txt | 185 ++--- .../tpch-approved-plan/v1-ras/spark33/13.txt | 199 ++--- .../tpch-approved-plan/v1-ras/spark33/14.txt | 132 ++-- .../tpch-approved-plan/v1-ras/spark33/15.txt | 256 ++++--- .../tpch-approved-plan/v1-ras/spark33/16.txt | 241 +++--- .../tpch-approved-plan/v1-ras/spark33/17.txt | 216 +++--- .../tpch-approved-plan/v1-ras/spark33/18.txt | 377 +++++----- .../tpch-approved-plan/v1-ras/spark33/19.txt | 128 ++-- .../tpch-approved-plan/v1-ras/spark33/20.txt | 494 ++++++------ .../tpch-approved-plan/v1-ras/spark33/21.txt | 467 ++++++------ .../tpch-approved-plan/v1-ras/spark33/22.txt | 243 +++--- .../tpch-approved-plan/v1-ras/spark33/3.txt | 230 +++--- .../tpch-approved-plan/v1-ras/spark33/4.txt | 189 ++--- .../tpch-approved-plan/v1-ras/spark33/5.txt | 533 +++++++------ .../tpch-approved-plan/v1-ras/spark33/6.txt | 67 +- .../tpch-approved-plan/v1-ras/spark33/7.txt | 506 +++++++------ .../tpch-approved-plan/v1-ras/spark33/8.txt | 709 ++++++++++-------- .../tpch-approved-plan/v1-ras/spark33/9.txt | 529 +++++++------ .../tpch-approved-plan/v1-ras/spark34/1.txt | 83 +- .../tpch-approved-plan/v1-ras/spark34/10.txt | 340 +++++---- .../tpch-approved-plan/v1-ras/spark34/11.txt | 473 ++++++------ .../tpch-approved-plan/v1-ras/spark34/12.txt | 185 ++--- .../tpch-approved-plan/v1-ras/spark34/13.txt | 199 ++--- .../tpch-approved-plan/v1-ras/spark34/14.txt | 132 ++-- .../tpch-approved-plan/v1-ras/spark34/15.txt | 256 ++++--- .../tpch-approved-plan/v1-ras/spark34/16.txt | 241 +++--- .../tpch-approved-plan/v1-ras/spark34/17.txt | 216 +++--- .../tpch-approved-plan/v1-ras/spark34/18.txt | 377 +++++----- .../tpch-approved-plan/v1-ras/spark34/19.txt | 128 ++-- .../tpch-approved-plan/v1-ras/spark34/20.txt | 494 ++++++------ .../tpch-approved-plan/v1-ras/spark34/21.txt | 467 ++++++------ .../tpch-approved-plan/v1-ras/spark34/22.txt | 243 +++--- .../tpch-approved-plan/v1-ras/spark34/3.txt | 230 +++--- .../tpch-approved-plan/v1-ras/spark34/4.txt | 189 ++--- .../tpch-approved-plan/v1-ras/spark34/5.txt | 533 +++++++------ .../tpch-approved-plan/v1-ras/spark34/6.txt | 67 +- .../tpch-approved-plan/v1-ras/spark34/7.txt | 506 +++++++------ .../tpch-approved-plan/v1-ras/spark34/8.txt | 709 ++++++++++-------- .../tpch-approved-plan/v1-ras/spark34/9.txt | 529 +++++++------ .../tpch-approved-plan/v1/spark32/1.txt | 83 +- .../tpch-approved-plan/v1/spark32/10.txt | 340 +++++---- .../tpch-approved-plan/v1/spark32/11.txt | 279 +++---- .../tpch-approved-plan/v1/spark32/12.txt | 185 ++--- .../tpch-approved-plan/v1/spark32/13.txt | 199 ++--- .../tpch-approved-plan/v1/spark32/14.txt | 132 ++-- .../tpch-approved-plan/v1/spark32/15.txt | 168 +++-- .../tpch-approved-plan/v1/spark32/16.txt | 241 +++--- .../tpch-approved-plan/v1/spark32/17.txt | 216 +++--- .../tpch-approved-plan/v1/spark32/18.txt | 377 +++++----- .../tpch-approved-plan/v1/spark32/19.txt | 128 ++-- .../tpch-approved-plan/v1/spark32/20.txt | 502 +++++++------ .../tpch-approved-plan/v1/spark32/21.txt | 471 ++++++------ .../tpch-approved-plan/v1/spark32/22.txt | 173 +++-- .../tpch-approved-plan/v1/spark32/3.txt | 230 +++--- .../tpch-approved-plan/v1/spark32/4.txt | 189 ++--- .../tpch-approved-plan/v1/spark32/5.txt | 533 +++++++------ .../tpch-approved-plan/v1/spark32/6.txt | 67 +- .../tpch-approved-plan/v1/spark32/7.txt | 506 +++++++------ .../tpch-approved-plan/v1/spark32/8.txt | 709 ++++++++++-------- .../tpch-approved-plan/v1/spark32/9.txt | 529 +++++++------ .../tpch-approved-plan/v1/spark33/1.txt | 83 +- .../tpch-approved-plan/v1/spark33/10.txt | 340 +++++---- .../tpch-approved-plan/v1/spark33/11.txt | 473 ++++++------ .../tpch-approved-plan/v1/spark33/12.txt | 185 ++--- .../tpch-approved-plan/v1/spark33/13.txt | 199 ++--- .../tpch-approved-plan/v1/spark33/14.txt | 132 ++-- .../tpch-approved-plan/v1/spark33/15.txt | 256 ++++--- .../tpch-approved-plan/v1/spark33/16.txt | 241 +++--- .../tpch-approved-plan/v1/spark33/17.txt | 216 +++--- .../tpch-approved-plan/v1/spark33/18.txt | 377 +++++----- .../tpch-approved-plan/v1/spark33/19.txt | 128 ++-- .../tpch-approved-plan/v1/spark33/20.txt | 494 ++++++------ .../tpch-approved-plan/v1/spark33/21.txt | 467 ++++++------ .../tpch-approved-plan/v1/spark33/22.txt | 275 +++---- .../tpch-approved-plan/v1/spark33/3.txt | 230 +++--- .../tpch-approved-plan/v1/spark33/4.txt | 189 ++--- .../tpch-approved-plan/v1/spark33/5.txt | 533 +++++++------ .../tpch-approved-plan/v1/spark33/6.txt | 67 +- .../tpch-approved-plan/v1/spark33/7.txt | 506 +++++++------ .../tpch-approved-plan/v1/spark33/8.txt | 709 ++++++++++-------- .../tpch-approved-plan/v1/spark33/9.txt | 529 +++++++------ .../tpch-approved-plan/v1/spark34/1.txt | 83 +- .../tpch-approved-plan/v1/spark34/10.txt | 340 +++++---- .../tpch-approved-plan/v1/spark34/11.txt | 473 ++++++------ .../tpch-approved-plan/v1/spark34/12.txt | 185 ++--- .../tpch-approved-plan/v1/spark34/13.txt | 199 ++--- .../tpch-approved-plan/v1/spark34/14.txt | 132 ++-- .../tpch-approved-plan/v1/spark34/15.txt | 256 ++++--- .../tpch-approved-plan/v1/spark34/16.txt | 241 +++--- .../tpch-approved-plan/v1/spark34/17.txt | 216 +++--- .../tpch-approved-plan/v1/spark34/18.txt | 377 +++++----- .../tpch-approved-plan/v1/spark34/19.txt | 128 ++-- .../tpch-approved-plan/v1/spark34/20.txt | 494 ++++++------ .../tpch-approved-plan/v1/spark34/21.txt | 467 ++++++------ .../tpch-approved-plan/v1/spark34/22.txt | 275 +++---- .../tpch-approved-plan/v1/spark34/3.txt | 230 +++--- .../tpch-approved-plan/v1/spark34/4.txt | 189 ++--- .../tpch-approved-plan/v1/spark34/5.txt | 533 +++++++------ .../tpch-approved-plan/v1/spark34/6.txt | 67 +- .../tpch-approved-plan/v1/spark34/7.txt | 506 +++++++------ .../tpch-approved-plan/v1/spark34/8.txt | 709 ++++++++++-------- .../tpch-approved-plan/v1/spark34/9.txt | 529 +++++++------ 252 files changed, 33344 insertions(+), 29294 deletions(-) diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt index 9316f6f8ebb50..69a930ea8545d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt index d67e373daf9b4..8f93964d76f50 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -194,7 +195,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -209,155 +210,159 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt index f65747c37e653..0e98c4c0edd72 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/11.txt @@ -1,55 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (58) + +- Exchange (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- BroadcastHashJoin Inner BuildRight (51) + :- Project (46) + : +- BroadcastHashJoin Inner BuildRight (45) + : :- Filter (41) + : : +- Scan parquet (40) + : +- BroadcastExchange (44) + : +- Filter (43) + : +- Scan parquet (42) + +- BroadcastExchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -187,124 +188,128 @@ Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery su Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(35) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(37) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(40) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(42) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(44) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(46) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(49) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(52) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(53) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(54) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(55) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(55) Filter +(56) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(57) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(58) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt index f1b7bbfe6acb4..d4942783d0fff 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/12.txt @@ -1,41 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- VeloxAppendBatches (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (42) + +- Exchange (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- BroadcastHashJoin Inner BuildLeft (36) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -131,96 +132,100 @@ Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_ Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(27) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(31) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(32) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(33) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(34) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(35) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(36) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(37) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(38) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(39) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(41) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt index 12147d4b3197e..4320ccecc9110 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/13.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- BroadcastHashJoin LeftOuter BuildRight (43) + :- Scan parquet (38) + +- BroadcastExchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -174,109 +175,113 @@ Results [2]: [c_count#X, count(1)#X AS custdist#X] Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(38) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(46) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(48) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(49) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(51) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt index 87dc45754b665..b79d92ba42577 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -85,7 +86,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] @@ -96,95 +97,99 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt index 0bfcd38f02d22..50fc5e0f66301 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/15.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- Project (41) - +- BroadcastHashJoin Inner BuildLeft (40) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Filter (39) - +- HashAggregate (38) - +- Exchange (37) - +- HashAggregate (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- Project (42) + +- BroadcastHashJoin Inner BuildLeft (41) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -137,100 +138,104 @@ Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_ Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) BroadcastExchange +(33) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(35) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(36) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(37) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(39) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(40) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(41) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(42) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt index ccc5488215a66..ed34336815320 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/16.txt @@ -1,53 +1,54 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- BroadcastHashJoin LeftAnti BuildRight (43) + : :- Filter (38) + : : +- Scan parquet (37) + : +- BroadcastExchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -175,134 +176,138 @@ Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_c Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(29) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(30) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(37) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(41) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(44) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(46) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(48) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(49) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(50) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(52) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(53) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(56) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt index 9d81e1d5053f9..1b258da06f206 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ RegularHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -47,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) FlushableHashAggregateExecTransformer +(5) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] @@ -58,141 +59,145 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt index e582fd47cf55d..505beebed8683 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ RegularHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -123,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -138,333 +140,341 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) NoopFilter +(31) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt index 72a9f74f86ced..f4c3b231f1ba7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -84,7 +85,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -95,91 +96,95 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt index e09911ea6ad8a..426a01b76051c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/20.txt @@ -1,97 +1,99 @@ == Physical Plan == -AdaptiveSparkPlan (107) +AdaptiveSparkPlan (109) +- == Final Plan == - VeloxColumnarToRowExec (70) - +- ^ SortExecTransformer (68) - +- ^ InputIteratorTransformer (67) - +- ShuffleQueryStage (65) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- AQEShuffleRead (7) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- BroadcastQueryStage (48) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ NoopFilter (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- BroadcastQueryStage (17) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ NoopFilter (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ShuffleQueryStage (38) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ NoopFilter (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- BroadcastQueryStage (30) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- BroadcastQueryStage (58) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ NoopFilter (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (72) + +- ^ SortExecTransformer (70) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67) + +- ColumnarExchange (66) + +- VeloxAppendBatches (65) + +- ^ ProjectExecTransformer (63) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (62) + :- ^ ProjectExecTransformer (53) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (52) + : :- ^ InputIteratorTransformer (10) + : : +- AQEShuffleRead (8) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (51) + : +- BroadcastQueryStage (49) + : +- ColumnarBroadcastExchange (48) + : +- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (45) + : :- ^ InputIteratorTransformer (26) + : : +- BroadcastQueryStage (24) + : : +- ColumnarBroadcastExchange (23) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) + : : :- ^ NoopFilter (12) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (20) + : : +- BroadcastQueryStage (18) + : : +- ColumnarBroadcastExchange (17) + : : +- ^ ProjectExecTransformer (15) + : : +- ^ NoopFilter (14) + : : +- ^ Scan parquet (13) + : +- ^ FilterExecTransformer (44) + : +- ^ ProjectExecTransformer (43) + : +- ^ RegularHashAggregateExecTransformer (42) + : +- ^ InputIteratorTransformer (41) + : +- ShuffleQueryStage (39) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ NoopFilter (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (61) + +- BroadcastQueryStage (59) + +- ColumnarBroadcastExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ NoopFilter (55) + +- ^ Scan parquet (54) +- == Initial Plan == - Sort (106) - +- Exchange (105) - +- Project (104) - +- BroadcastHashJoin Inner BuildRight (103) - :- Project (98) - : +- ShuffledHashJoin LeftSemi BuildRight (97) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (96) - : +- Project (95) - : +- BroadcastHashJoin Inner BuildLeft (94) - : :- BroadcastExchange (81) - : : +- BroadcastHashJoin LeftSemi BuildRight (80) - : : :- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (79) - : : +- Project (78) - : : +- Filter (77) - : : +- Scan parquet (76) - : +- Filter (93) - : +- HashAggregate (92) - : +- Exchange (91) - : +- HashAggregate (90) - : +- BroadcastHashJoin LeftSemi BuildRight (89) - : :- Project (84) - : : +- Filter (83) - : : +- Scan parquet (82) - : +- BroadcastExchange (88) - : +- Project (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (102) - +- Project (101) - +- Filter (100) - +- Scan parquet (99) + Sort (108) + +- Exchange (107) + +- Project (106) + +- BroadcastHashJoin Inner BuildRight (105) + :- Project (100) + : +- ShuffledHashJoin LeftSemi BuildRight (99) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Exchange (98) + : +- Project (97) + : +- BroadcastHashJoin Inner BuildLeft (96) + : :- BroadcastExchange (83) + : : +- BroadcastHashJoin LeftSemi BuildRight (82) + : : :- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (81) + : : +- Project (80) + : : +- Filter (79) + : : +- Scan parquet (78) + : +- Filter (95) + : +- HashAggregate (94) + : +- Exchange (93) + : +- HashAggregate (92) + : +- BroadcastHashJoin LeftSemi BuildRight (91) + : :- Project (86) + : : +- Filter (85) + : : +- Scan parquet (84) + : +- BroadcastExchange (90) + : +- Project (89) + : +- Filter (88) + : +- Scan parquet (87) + +- BroadcastExchange (104) + +- Project (103) + +- Filter (102) + +- Scan parquet (101) (1) Scan parquet @@ -113,448 +115,456 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(8) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(9) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(11) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) NoopFilter +(12) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) Scan parquet +(13) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) NoopFilter +(14) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(14) ProjectExecTransformer +(15) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(18) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(27) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) NoopFilter +(28) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(30) ReusedExchange [Reuses operator id: 17] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(31) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(34) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(35) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(38) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(39) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(42) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(44) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(45) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(48) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(49) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(50) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(51) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(52) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(53) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) NoopFilter +(55) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(55) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(58) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(59) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(60) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(62) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(63) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(65) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(66) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_address#X] -(67) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(68) SortExecTransformer +(70) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(69) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(70) VeloxColumnarToRowExec +(72) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(71) Scan parquet +(73) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(72) Filter +(74) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(73) Exchange +(75) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(76) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(75) Filter +(77) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(76) Scan parquet +(78) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(77) Filter +(79) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(78) Project +(80) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(79) BroadcastExchange +(81) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(81) BroadcastExchange +(83) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) Scan parquet +(84) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(83) Filter +(85) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(84) Project +(86) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(85) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(88) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(90) HashAggregate +(92) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(91) Exchange +(93) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(94) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Filter +(95) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(94) BroadcastHashJoin +(96) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(95) Project +(97) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(96) Exchange +(98) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(99) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(98) Project +(100) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(99) Scan parquet +(101) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(100) Filter +(102) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(101) Project +(103) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(102) BroadcastExchange +(104) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(103) BroadcastHashJoin +(105) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(106) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(105) Exchange +(107) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Sort +(108) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(107) AdaptiveSparkPlan +(109) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt index 10e08cfd60b57..10cf069932d74 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/21.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- TakeOrderedAndProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (75) - : : +- BroadcastHashJoin Inner BuildLeft (74) - : : :- BroadcastExchange (62) - : : : +- Filter (61) - : : : +- Scan parquet (60) - : : +- BroadcastHashJoin LeftAnti BuildRight (73) - : : :- BroadcastHashJoin LeftSemi BuildRight (68) - : : : :- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (67) - : : : +- Scan parquet (66) - : : +- BroadcastExchange (72) - : : +- Project (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- BroadcastExchange (79) - : +- Project (78) - : +- Filter (77) - : +- Scan parquet (76) - +- BroadcastExchange (85) - +- Project (84) - +- Filter (83) - +- Scan parquet (82) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (76) + : : +- BroadcastHashJoin Inner BuildLeft (75) + : : :- BroadcastExchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- BroadcastHashJoin LeftAnti BuildRight (74) + : : :- BroadcastHashJoin LeftSemi BuildRight (69) + : : : :- Project (66) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (68) + : : : +- Scan parquet (67) + : : +- BroadcastExchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- BroadcastExchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- BroadcastExchange (86) + +- Project (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -286,7 +287,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] @@ -301,194 +302,198 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(59) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(60) Scan parquet +(61) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(62) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) BroadcastExchange +(63) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(64) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(65) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) Project +(66) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(67) BroadcastExchange +(68) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(68) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(69) Scan parquet +(70) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(70) Filter +(71) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(71) Project +(72) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(72) BroadcastExchange +(73) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(74) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(75) Project +(76) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(76) Scan parquet +(77) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(77) Filter +(78) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(78) Project +(79) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(79) BroadcastExchange +(80) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(81) Project +(82) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(82) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(84) Project +(85) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(85) BroadcastExchange +(86) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(87) Project +(88) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(88) HashAggregate +(89) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(89) Exchange +(90) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(91) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(92) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt index a8c66bef1716b..287d6c30505ac 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/22.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- VeloxAppendBatches (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (38) + +- Exchange (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- BroadcastHashJoin LeftAnti BuildRight (32) + :- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (31) + +- Scan parquet (30) (1) Scan parquet @@ -118,87 +119,91 @@ Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacc Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(20) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(21) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(25) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(28) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(29) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(30) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(31) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(32) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(33) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(34) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(37) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(38) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt index b9b7951d76528..d92e785299d4c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ RegularHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -144,7 +145,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -159,131 +160,135 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt index 97d18724dc9d1..68bfa339986a3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/4.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- BroadcastHashJoin LeftSemi BuildRight (38) + :- Project (33) + : +- Filter (32) + : +- Scan parquet (31) + +- BroadcastExchange (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -137,100 +138,104 @@ Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(31) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(32) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(33) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(36) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(37) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(39) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(40) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(41) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(43) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt index a9b993d0a0780..612d61aac3b4c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/5.txt @@ -1,91 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (101) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (100) + +- Exchange (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Project (72) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -337,196 +338,200 @@ Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedpric Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(59) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(60) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(69) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(74) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(79) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(84) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(85) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(87) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(89) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(92) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(95) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(96) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(97) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(98) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(98) Exchange +(99) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(100) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(101) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt index 7aae3dccfd9ba..a5022343944f9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt index b0d68672ca3ac..3ed89ddff1621 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/7.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40) - +- ReusedExchange (39) + VeloxColumnarToRowExec (61) + +- ^ SortExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (83) + : +- BroadcastHashJoin Inner BuildRight (82) + : :- Project (78) + : : +- BroadcastHashJoin Inner BuildRight (77) + : : :- Project (73) + : : : +- BroadcastHashJoin Inner BuildRight (72) + : : : :- Project (68) + : : : : +- BroadcastHashJoin Inner BuildLeft (67) + : : : : :- BroadcastExchange (64) + : : : : : +- Filter (63) + : : : : : +- Scan parquet (62) + : : : : +- Filter (66) + : : : : +- Scan parquet (65) + : : : +- BroadcastExchange (71) + : : : +- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- BroadcastExchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- BroadcastExchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -307,188 +308,192 @@ Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(54) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(55) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(56) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(57) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(58) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(59) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(62) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(64) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(65) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(68) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(69) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(70) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(71) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(73) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(78) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(81) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(83) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(88) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(89) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(90) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(92) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(93) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt index 7e20b94edaa72..f309fbdce2590 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/8.txt @@ -1,116 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (130) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ShuffleQueryStage (80) + +- ColumnarExchange (79) + +- VeloxAppendBatches (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (129) + +- Exchange (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- BroadcastHashJoin Inner BuildRight (123) + :- Project (118) + : +- BroadcastHashJoin Inner BuildRight (117) + : :- Project (113) + : : +- BroadcastHashJoin Inner BuildRight (112) + : : :- Project (108) + : : : +- BroadcastHashJoin Inner BuildRight (107) + : : : :- Project (103) + : : : : +- BroadcastHashJoin Inner BuildRight (102) + : : : : :- Project (98) + : : : : : +- BroadcastHashJoin Inner BuildRight (97) + : : : : : :- Project (93) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) + : : : : : : :- BroadcastExchange (89) + : : : : : : : +- Project (88) + : : : : : : : +- Filter (87) + : : : : : : : +- Scan parquet (86) + : : : : : : +- Filter (91) + : : : : : : +- Scan parquet (90) + : : : : : +- BroadcastExchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- BroadcastExchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- BroadcastExchange (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- BroadcastExchange (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- BroadcastExchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- BroadcastExchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -442,244 +443,248 @@ Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.000 Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(78) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(79) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(80) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(81) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(82) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(83) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(90) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(93) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(94) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(95) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(96) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(98) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(101) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(103) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(104) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(105) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(106) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(107) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(108) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(111) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(112) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(113) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(114) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(116) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(117) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(118) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(119) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(121) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(122) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(123) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(124) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(125) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(126) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(127) Exchange +(128) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(129) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(130) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt index 8a30fbdfb1520..0420b98c2c1f5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark32/9.txt @@ -1,89 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (98) + +- Exchange (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (69) + : : : : : +- Project (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -331,192 +332,196 @@ Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(59) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(63) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(73) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(78) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(80) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(83) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(88) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(91) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(93) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(94) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(97) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(98) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt index 63b30bb5d26b0..5ad7bc9d3675d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt index 18c2ef9bc4731..4e657eb7e540c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -194,7 +195,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -209,155 +210,159 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt index 6afbee587ed27..5be334bc198ea 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/11.txt @@ -1,55 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (58) + +- Exchange (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- BroadcastHashJoin Inner BuildRight (51) + :- Project (46) + : +- BroadcastHashJoin Inner BuildRight (45) + : :- Filter (41) + : : +- Scan parquet (40) + : +- BroadcastExchange (44) + : +- Filter (43) + : +- Scan parquet (42) + +- BroadcastExchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -187,350 +188,354 @@ Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery su Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(35) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(37) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(40) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(42) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(44) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(46) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(49) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(52) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(53) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(54) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(55) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(55) Filter +(56) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(57) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(58) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ NoopFilter (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (83) + +- ^ ProjectExecTransformer (81) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ShuffleQueryStage (77), Statistics(X) + +- ColumnarExchange (76) + +- ^ FlushableHashAggregateExecTransformer (74) + +- ^ ProjectExecTransformer (73) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (72) + :- ^ ProjectExecTransformer (67) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + : :- ^ NoopFilter (61) + : : +- ^ Scan parquet (60) + : +- ^ InputIteratorTransformer (65) + : +- BroadcastQueryStage (63), Statistics(X) + : +- ReusedExchange (62) + +- ^ InputIteratorTransformer (71) + +- BroadcastQueryStage (69), Statistics(X) + +- ReusedExchange (68) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Filter (85) + : : +- Scan parquet (84) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) + + +(60) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) NoopFilter +(61) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(61) ReusedExchange [Reuses operator id: 6] +(62) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(63) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(64) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(65) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(66) ProjectExecTransformer +(67) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(68) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(69) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(70) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(71) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(72) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(74) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(76) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(77) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(78) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(79) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(80) ProjectExecTransformer +(81) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(81) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(84) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(86) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(89) Project +(90) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(93) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(95) Project +(96) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(97) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt index c65916d668988..7104a3e8bc38f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/12.txt @@ -1,41 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- VeloxAppendBatches (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (42) + +- Exchange (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- BroadcastHashJoin Inner BuildLeft (36) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -131,96 +132,100 @@ Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_ Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(27) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(31) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(32) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(33) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(34) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(35) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(36) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(37) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(38) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(39) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(41) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt index 9ff054f9314e1..9658b4be0cc70 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/13.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- BroadcastHashJoin LeftOuter BuildRight (43) + :- Scan parquet (38) + +- BroadcastExchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -174,109 +175,113 @@ Results [2]: [c_count#X, count(1)#X AS custdist#X] Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(38) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(46) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(48) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(49) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(51) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt index cc14a0347f1d0..fbcaf99758c48 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -85,7 +86,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] @@ -96,95 +97,99 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt index 2d463a561ab6d..91b117a86c00b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/15.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- Project (39) + +- BroadcastHashJoin Inner BuildLeft (38) + :- BroadcastExchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -136,246 +137,250 @@ Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_ Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(36) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(38) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ NoopFilter (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ NoopFilter (44) + +- ^ Scan parquet (43) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(42) Scan parquet +(43) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) NoopFilter +(44) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(46) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(49) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(53) ProjectExecTransformer +(54) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(59) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(60) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(61) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(62) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(63) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(64) HashAggregate +(65) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(66) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt index c4d21c42e70b9..da533a30dde74 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/16.txt @@ -1,53 +1,54 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- BroadcastHashJoin LeftAnti BuildRight (43) + : :- Filter (38) + : : +- Scan parquet (37) + : +- BroadcastExchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -175,134 +176,138 @@ Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_c Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(29) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(30) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(37) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(41) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(44) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(46) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(48) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(49) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(50) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(52) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(53) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(56) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt index 0b41400d5864d..a57c908f01784 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ RegularHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -47,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) FlushableHashAggregateExecTransformer +(5) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] @@ -58,141 +59,145 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt index 8f03bd54cd2d8..810b57c0f9bb5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ RegularHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -123,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -138,333 +140,341 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) NoopFilter +(31) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt index 1fcd294716efe..46bbc58abcde7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -84,7 +85,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -95,91 +96,95 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt index ee30d646767e2..3ac12224cffab 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/20.txt @@ -1,96 +1,98 @@ == Physical Plan == -AdaptiveSparkPlan (104) +AdaptiveSparkPlan (106) +- == Final Plan == - VeloxColumnarToRowExec (67) - +- AQEShuffleRead (66) - +- ShuffleQueryStage (65), Statistics(X) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- AQEShuffleRead (7) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- BroadcastQueryStage (48), Statistics(X) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ NoopFilter (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- BroadcastQueryStage (17), Statistics(X) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ NoopFilter (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ShuffleQueryStage (38), Statistics(X) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ NoopFilter (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- BroadcastQueryStage (30), Statistics(X) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- BroadcastQueryStage (58), Statistics(X) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ NoopFilter (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (69) + +- AQEShuffleRead (68) + +- ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- VeloxAppendBatches (65) + +- ^ ProjectExecTransformer (63) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (62) + :- ^ ProjectExecTransformer (53) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (52) + : :- ^ InputIteratorTransformer (10) + : : +- AQEShuffleRead (8) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (51) + : +- BroadcastQueryStage (49), Statistics(X) + : +- ColumnarBroadcastExchange (48) + : +- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (45) + : :- ^ InputIteratorTransformer (26) + : : +- BroadcastQueryStage (24), Statistics(X) + : : +- ColumnarBroadcastExchange (23) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) + : : :- ^ NoopFilter (12) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (20) + : : +- BroadcastQueryStage (18), Statistics(X) + : : +- ColumnarBroadcastExchange (17) + : : +- ^ ProjectExecTransformer (15) + : : +- ^ NoopFilter (14) + : : +- ^ Scan parquet (13) + : +- ^ FilterExecTransformer (44) + : +- ^ ProjectExecTransformer (43) + : +- ^ RegularHashAggregateExecTransformer (42) + : +- ^ InputIteratorTransformer (41) + : +- ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ NoopFilter (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31), Statistics(X) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (61) + +- BroadcastQueryStage (59), Statistics(X) + +- ColumnarBroadcastExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ NoopFilter (55) + +- ^ Scan parquet (54) +- == Initial Plan == - Sort (103) - +- Exchange (102) - +- Project (101) - +- BroadcastHashJoin Inner BuildRight (100) - :- Project (95) - : +- ShuffledHashJoin LeftSemi BuildRight (94) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Exchange (93) - : +- Project (92) - : +- BroadcastHashJoin Inner BuildLeft (91) - : :- BroadcastExchange (78) - : : +- BroadcastHashJoin LeftSemi BuildRight (77) - : : :- Filter (72) - : : : +- Scan parquet (71) - : : +- BroadcastExchange (76) - : : +- Project (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Filter (90) - : +- HashAggregate (89) - : +- Exchange (88) - : +- HashAggregate (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Project (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (85) - : +- Project (84) - : +- Filter (83) - : +- Scan parquet (82) - +- BroadcastExchange (99) - +- Project (98) - +- Filter (97) - +- Scan parquet (96) + Sort (105) + +- Exchange (104) + +- Project (103) + +- BroadcastHashJoin Inner BuildRight (102) + :- Project (97) + : +- ShuffledHashJoin LeftSemi BuildRight (96) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (95) + : +- Project (94) + : +- BroadcastHashJoin Inner BuildLeft (93) + : :- BroadcastExchange (80) + : : +- BroadcastHashJoin LeftSemi BuildRight (79) + : : :- Filter (74) + : : : +- Scan parquet (73) + : : +- BroadcastExchange (78) + : : +- Project (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- Filter (92) + : +- HashAggregate (91) + : +- Exchange (90) + : +- HashAggregate (89) + : +- BroadcastHashJoin LeftSemi BuildRight (88) + : :- Project (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (87) + : +- Project (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (101) + +- Project (100) + +- Filter (99) + +- Scan parquet (98) (1) Scan parquet @@ -112,438 +114,446 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(8) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(9) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(11) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) NoopFilter +(12) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) Scan parquet +(13) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) NoopFilter +(14) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(14) ProjectExecTransformer +(15) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(18) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(27) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) NoopFilter +(28) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(30) ReusedExchange [Reuses operator id: 17] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(31) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(34) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(35) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(38) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(39) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(42) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(44) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(45) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(48) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(49) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(50) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(51) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(52) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(53) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) NoopFilter +(55) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(55) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(58) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(59) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(60) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(62) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(63) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(65) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(66) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) AQEShuffleRead +(68) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(67) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(68) Scan parquet +(70) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(69) Filter +(71) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(70) Exchange +(72) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(73) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(72) Filter +(74) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(73) Scan parquet +(75) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(75) Project +(77) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(76) BroadcastExchange +(78) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(77) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(78) BroadcastExchange +(80) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(79) Scan parquet +(81) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(80) Filter +(82) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(81) Project +(83) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(82) Scan parquet +(84) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(83) Filter +(85) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(84) Project +(86) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(85) BroadcastExchange +(87) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(87) HashAggregate +(89) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(88) Exchange +(90) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(91) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(90) Filter +(92) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(92) Project +(94) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Exchange +(95) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(95) Project +(97) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(96) Scan parquet +(98) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(97) Filter +(99) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(98) Project +(100) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(99) BroadcastExchange +(101) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(100) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(101) Project +(103) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(102) Exchange +(104) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Sort +(105) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(104) AdaptiveSparkPlan +(106) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt index c52c51139d2ad..9f48891225fb8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/21.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -285,7 +286,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] @@ -300,190 +301,194 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(74) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(80) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt index d404e2149a89a..404a3debc42cb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/22.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- VeloxAppendBatches (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (38) + +- Exchange (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- BroadcastHashJoin LeftAnti BuildRight (32) + :- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (31) + +- Scan parquet (30) (1) Scan parquet @@ -118,201 +119,205 @@ Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacc Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(20) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(21) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(25) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(28) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(29) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(30) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(31) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(32) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(33) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(34) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(37) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(38) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (51) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- Filter (53) + +- Scan parquet (52) -(39) Scan parquet +(40) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) NoopFilter +(41) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(41) ProjectExecTransformer +(42) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(45) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(46) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(47) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(48) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(51) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(52) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(53) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(54) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(55) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(56) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(57) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt index dbe67ba34d042..aacc67b90b3f9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ RegularHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -144,7 +145,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -159,131 +160,135 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt index 626f608cecd07..5f3ea7038ab0b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/4.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- BroadcastHashJoin LeftSemi BuildRight (38) + :- Project (33) + : +- Filter (32) + : +- Scan parquet (31) + +- BroadcastExchange (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -137,100 +138,104 @@ Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(31) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(32) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(33) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(36) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(37) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(39) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(40) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(41) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(43) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt index 7a2ce7cf7a4a2..d83f2507a1002 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/5.txt @@ -1,91 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (101) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (100) + +- Exchange (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Project (72) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -337,196 +338,200 @@ Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedpric Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(59) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(60) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(69) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(74) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(79) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(84) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(85) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(87) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(89) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(92) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(95) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(96) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(97) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(98) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(98) Exchange +(99) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(100) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(101) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt index e03830992c2e2..30e1b3fe013b5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt index 4f9b5f20956d3..24cd98650a171 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/7.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (61) + +- ^ SortExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (83) + : +- BroadcastHashJoin Inner BuildRight (82) + : :- Project (78) + : : +- BroadcastHashJoin Inner BuildRight (77) + : : :- Project (73) + : : : +- BroadcastHashJoin Inner BuildRight (72) + : : : :- Project (68) + : : : : +- BroadcastHashJoin Inner BuildLeft (67) + : : : : :- BroadcastExchange (64) + : : : : : +- Filter (63) + : : : : : +- Scan parquet (62) + : : : : +- Filter (66) + : : : : +- Scan parquet (65) + : : : +- BroadcastExchange (71) + : : : +- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- BroadcastExchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- BroadcastExchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -307,188 +308,192 @@ Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(54) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(55) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(56) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(57) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(58) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(59) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(62) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(64) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(65) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(68) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(69) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(70) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(71) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(73) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(78) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(81) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(83) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(88) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(89) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(90) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(92) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(93) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt index 677a42eb711aa..8185bc4a349e5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/8.txt @@ -1,116 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (130) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ShuffleQueryStage (80), Statistics(X) + +- ColumnarExchange (79) + +- VeloxAppendBatches (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (129) + +- Exchange (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- BroadcastHashJoin Inner BuildRight (123) + :- Project (118) + : +- BroadcastHashJoin Inner BuildRight (117) + : :- Project (113) + : : +- BroadcastHashJoin Inner BuildRight (112) + : : :- Project (108) + : : : +- BroadcastHashJoin Inner BuildRight (107) + : : : :- Project (103) + : : : : +- BroadcastHashJoin Inner BuildRight (102) + : : : : :- Project (98) + : : : : : +- BroadcastHashJoin Inner BuildRight (97) + : : : : : :- Project (93) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) + : : : : : : :- BroadcastExchange (89) + : : : : : : : +- Project (88) + : : : : : : : +- Filter (87) + : : : : : : : +- Scan parquet (86) + : : : : : : +- Filter (91) + : : : : : : +- Scan parquet (90) + : : : : : +- BroadcastExchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- BroadcastExchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- BroadcastExchange (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- BroadcastExchange (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- BroadcastExchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- BroadcastExchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -442,244 +443,248 @@ Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.000 Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(78) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(79) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(80) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(81) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(82) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(83) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(90) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(93) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(94) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(95) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(96) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(98) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(101) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(103) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(104) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(105) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(106) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(107) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(108) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(111) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(112) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(113) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(114) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(116) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(117) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(118) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(119) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(121) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(122) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(123) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(124) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(125) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(126) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(127) Exchange +(128) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(129) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(130) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt index 9f2111b015caa..0d27f28fda4d5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark33/9.txt @@ -1,89 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (98) + +- Exchange (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (69) + : : : : : +- Project (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -331,192 +332,196 @@ Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(59) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(63) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(73) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(78) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(80) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(83) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(88) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(91) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(93) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(94) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(97) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(98) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt index c254ec8c82caf..589019c4cdbf5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt index ffc37f5b79081..f83547f843ca6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ NoopFilter (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ NoopFilter (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ NoopFilter (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ NoopFilter (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ NoopFilter (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ NoopFilter (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -197,7 +198,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -212,158 +213,162 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt index 6410bb3fed5f0..da2efbcb31343 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/11.txt @@ -1,55 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ NoopFilter (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ NoopFilter (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ NoopFilter (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ NoopFilter (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (58) + +- Exchange (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- BroadcastHashJoin Inner BuildRight (51) + :- Project (46) + : +- BroadcastHashJoin Inner BuildRight (45) + : :- Filter (41) + : : +- Scan parquet (40) + : +- BroadcastExchange (44) + : +- Filter (43) + : +- Scan parquet (42) + +- BroadcastExchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -189,356 +190,360 @@ Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery su Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(35) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(37) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(40) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(42) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(44) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(45) Project +(46) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(49) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(51) Project +(52) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(53) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(54) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(55) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(55) Filter +(56) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(57) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(58) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ NoopFilter (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (83) + +- ^ ProjectExecTransformer (81) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ShuffleQueryStage (77), Statistics(X) + +- ColumnarExchange (76) + +- ^ FlushableHashAggregateExecTransformer (74) + +- ^ ProjectExecTransformer (73) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (72) + :- ^ ProjectExecTransformer (67) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + : :- ^ NoopFilter (61) + : : +- ^ Scan parquet (60) + : +- ^ InputIteratorTransformer (65) + : +- BroadcastQueryStage (63), Statistics(X) + : +- ReusedExchange (62) + +- ^ InputIteratorTransformer (71) + +- BroadcastQueryStage (69), Statistics(X) + +- ReusedExchange (68) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Filter (85) + : : +- Scan parquet (84) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) + + +(60) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) NoopFilter +(61) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(61) ReusedExchange [Reuses operator id: 6] +(62) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(63) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(64) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(65) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(66) ProjectExecTransformer +(67) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(68) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(69) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(70) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(71) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(72) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(74) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(76) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(77) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(78) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(79) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(80) ProjectExecTransformer +(81) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(81) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(84) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(86) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(89) Project +(90) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(93) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(95) Project +(96) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(97) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt index 4e85d81940983..6006564bf2a8b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/12.txt @@ -1,41 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- VeloxAppendBatches (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (42) + +- Exchange (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- BroadcastHashJoin Inner BuildLeft (36) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -132,97 +133,101 @@ Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_ Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(27) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(31) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(32) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(33) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(34) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(35) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(36) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(36) Project +(37) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(38) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(39) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(41) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt index e83ab1aca83cb..434a767c15a48 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/13.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ NoopFilter (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ NoopFilter (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- BroadcastHashJoin LeftOuter BuildRight (43) + :- Scan parquet (38) + +- BroadcastExchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -175,110 +176,114 @@ Results [2]: [c_count#X, count(1)#X AS custdist#X] Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(38) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(46) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(48) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(49) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(51) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt index 58d532222af58..e4a7a29b15b07 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -86,7 +87,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] @@ -97,96 +98,100 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt index 2699216024690..f088b241bdd5e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/15.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ NoopFilter (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- Project (39) + +- BroadcastHashJoin Inner BuildLeft (38) + :- BroadcastExchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -137,247 +138,251 @@ Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_ Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(36) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(38) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ NoopFilter (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ NoopFilter (44) + +- ^ Scan parquet (43) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(42) Scan parquet +(43) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) NoopFilter +(44) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(46) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(49) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(53) ProjectExecTransformer +(54) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(59) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(60) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(61) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(62) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(63) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(64) HashAggregate +(65) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(66) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt index 0a87716190205..3fff24e2b4979 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/16.txt @@ -1,53 +1,54 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ NoopFilter (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ NoopFilter (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- BroadcastHashJoin LeftAnti BuildRight (43) + : :- Filter (38) + : : +- Scan parquet (37) + : +- BroadcastExchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -176,136 +177,140 @@ Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_c Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(29) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(30) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(37) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(41) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(43) Scan parquet +(44) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(46) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(47) Project +(48) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(49) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(50) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(52) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(53) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(56) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt index 62612096db425..e499293269fc8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ RegularHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -47,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) FlushableHashAggregateExecTransformer +(5) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] @@ -58,143 +59,147 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt index f0420b98194df..3408aba7dcbc4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ NoopFilter (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ RegularHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ NoopFilter (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -123,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -138,341 +140,349 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) NoopFilter +(31) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt index c39011756bca2..7fa7f7108a17d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -85,7 +86,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -96,92 +97,96 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt index 6bad140301d87..a4be44f50e60a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/20.txt @@ -1,90 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (96) +AdaptiveSparkPlan (97) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- AQEShuffleRead (59) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) - :- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (43) - : +- BroadcastQueryStage (41), Statistics(X) - : +- ColumnarBroadcastExchange (40) - : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) - : :- ^ InputIteratorTransformer (18) - : : +- BroadcastQueryStage (16), Statistics(X) - : : +- ColumnarBroadcastExchange (15) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) - : : :- ^ NoopFilter (4) - : : : +- ^ Scan parquet (3) - : : +- ^ InputIteratorTransformer (12) - : : +- BroadcastQueryStage (10), Statistics(X) - : : +- ColumnarBroadcastExchange (9) - : : +- ^ ProjectExecTransformer (7) - : : +- ^ NoopFilter (6) - : : +- ^ Scan parquet (5) - : +- ^ FilterExecTransformer (36) - : +- ^ ProjectExecTransformer (35) - : +- ^ RegularHashAggregateExecTransformer (34) - : +- ^ InputIteratorTransformer (33) - : +- ShuffleQueryStage (31), Statistics(X) - : +- ColumnarExchange (30) - : +- ^ ProjectExecTransformer (28) - : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ ProjectExecTransformer (21) - : : +- ^ NoopFilter (20) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ReusedExchange (22) - +- ^ InputIteratorTransformer (53) - +- BroadcastQueryStage (51), Statistics(X) - +- ColumnarBroadcastExchange (50) - +- ^ ProjectExecTransformer (48) - +- ^ NoopFilter (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (61) + +- AQEShuffleRead (60) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- VeloxAppendBatches (57) + +- ^ ProjectExecTransformer (55) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) + :- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (43) + : +- BroadcastQueryStage (41), Statistics(X) + : +- ColumnarBroadcastExchange (40) + : +- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) + : :- ^ InputIteratorTransformer (18) + : : +- BroadcastQueryStage (16), Statistics(X) + : : +- ColumnarBroadcastExchange (15) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) + : : :- ^ NoopFilter (4) + : : : +- ^ Scan parquet (3) + : : +- ^ InputIteratorTransformer (12) + : : +- BroadcastQueryStage (10), Statistics(X) + : : +- ColumnarBroadcastExchange (9) + : : +- ^ ProjectExecTransformer (7) + : : +- ^ NoopFilter (6) + : : +- ^ Scan parquet (5) + : +- ^ FilterExecTransformer (36) + : +- ^ ProjectExecTransformer (35) + : +- ^ RegularHashAggregateExecTransformer (34) + : +- ^ InputIteratorTransformer (33) + : +- ShuffleQueryStage (31), Statistics(X) + : +- ColumnarExchange (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) + +- ^ InputIteratorTransformer (53) + +- BroadcastQueryStage (51), Statistics(X) + +- ColumnarBroadcastExchange (50) + +- ^ ProjectExecTransformer (48) + +- ^ NoopFilter (47) + +- ^ Scan parquet (46) +- == Initial Plan == - Sort (95) - +- Exchange (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Filter (62) - : : +- Scan parquet (61) - : +- BroadcastExchange (85) - : +- Project (84) - : +- BroadcastHashJoin Inner BuildLeft (83) - : :- BroadcastExchange (70) - : : +- BroadcastHashJoin LeftSemi BuildRight (69) - : : :- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (68) - : : +- Project (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Filter (82) - : +- HashAggregate (81) - : +- Exchange (80) - : +- HashAggregate (79) - : +- BroadcastHashJoin LeftSemi BuildRight (78) - : :- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- BroadcastExchange (77) - : +- Project (76) - : +- Filter (75) - : +- Scan parquet (74) - +- BroadcastExchange (91) - +- Project (90) - +- Filter (89) - +- Scan parquet (88) + Sort (96) + +- Exchange (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (88) + : +- BroadcastHashJoin LeftSemi BuildRight (87) + : :- Filter (63) + : : +- Scan parquet (62) + : +- BroadcastExchange (86) + : +- Project (85) + : +- BroadcastHashJoin Inner BuildLeft (84) + : :- BroadcastExchange (71) + : : +- BroadcastHashJoin LeftSemi BuildRight (70) + : : :- Filter (65) + : : : +- Scan parquet (64) + : : +- BroadcastExchange (69) + : : +- Project (68) + : : +- Filter (67) + : : +- Scan parquet (66) + : +- Filter (83) + : +- HashAggregate (82) + : +- Exchange (81) + : +- HashAggregate (80) + : +- BroadcastHashJoin LeftSemi BuildRight (79) + : :- Project (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- BroadcastExchange (78) + : +- Project (77) + : +- Filter (76) + : +- Scan parquet (75) + +- BroadcastExchange (92) + +- Project (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -329,195 +330,199 @@ Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] Input [2]: [s_name#X, s_address#X] Arguments: false -(57) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(59) AQEShuffleRead +(60) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(61) Scan parquet +(62) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(63) Scan parquet +(64) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(64) Filter +(65) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) BroadcastHashJoin +(70) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(70) BroadcastExchange +(71) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(71) Scan parquet +(72) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(72) Filter +(73) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(73) Project +(74) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(76) Project +(77) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(77) BroadcastExchange +(78) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(79) HashAggregate +(80) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(80) Exchange +(81) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(82) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(82) Filter +(83) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(83) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(84) Project +(85) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(85) BroadcastExchange +(86) BroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(87) Project +(88) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(90) Project +(91) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(91) BroadcastExchange +(92) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) Project +(94) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(94) Exchange +(95) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Sort +(96) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(96) AdaptiveSparkPlan +(97) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt index 14b23aa966e51..89f395ed56d5a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/21.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ NoopFilter (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ NoopFilter (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ NoopFilter (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ NoopFilter (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ NoopFilter (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -290,7 +291,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] @@ -305,195 +306,199 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(74) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(80) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(86) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt index 4517787bcaef3..388bd109476a0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/22.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- VeloxAppendBatches (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (38) + +- Exchange (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- BroadcastHashJoin LeftAnti BuildRight (32) + :- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (31) + +- Scan parquet (30) (1) Scan parquet @@ -119,202 +120,206 @@ Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacc Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(20) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(21) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(25) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(28) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(29) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(30) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(31) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(32) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(32) Project +(33) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(34) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(37) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(38) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (51) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- Filter (53) + +- Scan parquet (52) -(39) Scan parquet +(40) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) NoopFilter +(41) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(41) ProjectExecTransformer +(42) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(45) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(46) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(47) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(48) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(51) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(52) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(53) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(54) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(55) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(56) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(57) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt index 361ea070d42ac..c157058fc2efa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ NoopFilter (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ RegularHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ NoopFilter (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ NoopFilter (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -146,7 +147,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -161,133 +162,137 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt index 4bdcd640058c2..5e3f77bb06daf 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/4.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ NoopFilter (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ NoopFilter (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- BroadcastHashJoin LeftSemi BuildRight (38) + :- Project (33) + : +- Filter (32) + : +- Scan parquet (31) + +- BroadcastExchange (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -138,101 +139,105 @@ Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(31) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(32) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(33) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(36) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(37) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(38) Project +(39) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(40) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(41) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(43) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt index d1f0cb5eba565..0f5d19f82450f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/5.txt @@ -1,91 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (101) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (100) + +- Exchange (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Project (72) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -342,201 +343,205 @@ Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS reven Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(59) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(60) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(69) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(73) Project +(74) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(78) Project +(79) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(83) Project +(84) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(85) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(87) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) Project +(89) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(92) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(94) Project +(95) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(96) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(97) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(98) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(98) Exchange +(99) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(100) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(101) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt index 5987a808f5fd8..fbcc75a247d54 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt index 29c2524a7615c..7a9949d5358ca 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/7.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ NoopFilter (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ NoopFilter (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ NoopFilter (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (61) + +- ^ SortExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ NoopFilter (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ NoopFilter (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ NoopFilter (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (83) + : +- BroadcastHashJoin Inner BuildRight (82) + : :- Project (78) + : : +- BroadcastHashJoin Inner BuildRight (77) + : : :- Project (73) + : : : +- BroadcastHashJoin Inner BuildRight (72) + : : : :- Project (68) + : : : : +- BroadcastHashJoin Inner BuildLeft (67) + : : : : :- BroadcastExchange (64) + : : : : : +- Filter (63) + : : : : : +- Scan parquet (62) + : : : : +- Filter (66) + : : : : +- Scan parquet (65) + : : : +- BroadcastExchange (71) + : : : +- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- BroadcastExchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- BroadcastExchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -312,193 +313,197 @@ Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(54) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(55) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(56) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(57) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(58) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(59) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(62) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(64) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(65) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(67) Project +(68) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(69) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(70) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(71) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(72) Project +(73) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(77) Project +(78) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(81) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(83) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(88) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(89) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(90) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(92) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(93) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt index 715bc6f4e5f88..e5fd2360e7589 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/8.txt @@ -1,116 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (130) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ NoopFilter (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ NoopFilter (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ NoopFilter (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ NoopFilter (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ NoopFilter (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ NoopFilter (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ShuffleQueryStage (80), Statistics(X) + +- ColumnarExchange (79) + +- VeloxAppendBatches (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ NoopFilter (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ NoopFilter (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ NoopFilter (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ NoopFilter (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ NoopFilter (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ NoopFilter (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ NoopFilter (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (129) + +- Exchange (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- BroadcastHashJoin Inner BuildRight (123) + :- Project (118) + : +- BroadcastHashJoin Inner BuildRight (117) + : :- Project (113) + : : +- BroadcastHashJoin Inner BuildRight (112) + : : :- Project (108) + : : : +- BroadcastHashJoin Inner BuildRight (107) + : : : :- Project (103) + : : : : +- BroadcastHashJoin Inner BuildRight (102) + : : : : :- Project (98) + : : : : : +- BroadcastHashJoin Inner BuildRight (97) + : : : : : :- Project (93) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) + : : : : : : :- BroadcastExchange (89) + : : : : : : : +- Project (88) + : : : : : : : +- Filter (87) + : : : : : : : +- Scan parquet (86) + : : : : : : +- Filter (91) + : : : : : : +- Scan parquet (90) + : : : : : +- BroadcastExchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- BroadcastExchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- BroadcastExchange (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- BroadcastExchange (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- BroadcastExchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- BroadcastExchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -449,251 +450,255 @@ Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.000 Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(78) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(79) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(80) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(81) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(82) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(83) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(90) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(92) Project +(93) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(94) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(95) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(96) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(97) Project +(98) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(101) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(102) Project +(103) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(104) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(105) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(106) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(107) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(107) Project +(108) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(111) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(112) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(112) Project +(113) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(114) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(116) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(117) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(118) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(119) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(121) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(122) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(123) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(123) Project +(124) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(125) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(126) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(127) Exchange +(128) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(129) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(130) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt index 14c32b04eb486..4538df9c485ab 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj-ras/spark34/9.txt @@ -1,89 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ NoopFilter (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ NoopFilter (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ NoopFilter (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ NoopFilter (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ NoopFilter (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ NoopFilter (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ NoopFilter (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ NoopFilter (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ NoopFilter (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (98) + +- Exchange (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (69) + : : : : : +- Project (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -336,197 +337,201 @@ Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(59) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(63) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(72) Project +(73) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(77) Project +(78) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(80) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(82) Project +(83) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(87) Project +(88) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(91) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(92) Project +(93) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(94) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(97) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(98) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt index 656f98574483b..1c8670c0fea4d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt index 3afadbc4c6787..61a6de21a4003 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -194,7 +195,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -209,155 +210,159 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt index 03199cd9feb1b..72b0287a9dbe5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/11.txt @@ -1,55 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (58) + +- Exchange (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- BroadcastHashJoin Inner BuildRight (51) + :- Project (46) + : +- BroadcastHashJoin Inner BuildRight (45) + : :- Filter (41) + : : +- Scan parquet (40) + : +- BroadcastExchange (44) + : +- Filter (43) + : +- Scan parquet (42) + +- BroadcastExchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -187,124 +188,128 @@ Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery su Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(35) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(37) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(40) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(42) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(44) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(46) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(49) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(52) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(53) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(54) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(55) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(55) Filter +(56) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(57) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(58) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt index e78891a522ff9..23ab061662fc7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/12.txt @@ -1,41 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- VeloxAppendBatches (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (42) + +- Exchange (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- BroadcastHashJoin Inner BuildLeft (36) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -131,96 +132,100 @@ Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_ Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(27) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(31) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(32) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(33) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(34) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(35) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(36) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(37) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(38) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(39) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(41) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt index 9aa658b2d78d9..0f55c5334af5d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/13.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- BroadcastHashJoin LeftOuter BuildRight (43) + :- Scan parquet (38) + +- BroadcastExchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -174,109 +175,113 @@ Results [2]: [c_count#X, count(1)#X AS custdist#X] Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(38) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(46) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(48) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(49) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(51) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt index fcd550bdb92c6..2427b05691d0b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -85,7 +86,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] @@ -96,95 +97,99 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt index 54dad76174f98..64f59fa9c8116 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/15.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- Project (41) - +- BroadcastHashJoin Inner BuildLeft (40) - :- BroadcastExchange (32) - : +- Filter (31) - : +- Scan parquet (30) - +- Filter (39) - +- HashAggregate (38) - +- Exchange (37) - +- HashAggregate (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- Project (42) + +- BroadcastHashJoin Inner BuildLeft (41) + :- BroadcastExchange (33) + : +- Filter (32) + : +- Scan parquet (31) + +- Filter (40) + +- HashAggregate (39) + +- Exchange (38) + +- HashAggregate (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -137,100 +138,104 @@ Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_ Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(30) Scan parquet +(31) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(32) BroadcastExchange +(33) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(33) Scan parquet +(34) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(34) Filter +(35) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(35) Project +(36) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(36) HashAggregate +(37) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(37) Exchange +(38) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(39) Filter +(40) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(40) BroadcastHashJoin +(41) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(41) Project +(42) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(42) Exchange +(43) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt index 5197f57218a42..7411f2a6667a7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/16.txt @@ -1,53 +1,54 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- BroadcastHashJoin LeftAnti BuildRight (43) + : :- Filter (38) + : : +- Scan parquet (37) + : +- BroadcastExchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -175,134 +176,138 @@ Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_c Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(29) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(30) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(37) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(41) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(44) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(46) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(48) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(49) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(50) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(52) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(53) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(56) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt index 9d81e1d5053f9..1b258da06f206 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ RegularHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -47,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) FlushableHashAggregateExecTransformer +(5) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] @@ -58,141 +59,145 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt index 3415895657402..ced5a3b68cc45 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ RegularHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -123,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -138,333 +140,341 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt index a46daacbdb75e..02f21577ad17d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -84,7 +85,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -95,91 +96,95 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt index 6fcedbbda9966..c5b3f4cc9283f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/20.txt @@ -1,97 +1,99 @@ == Physical Plan == -AdaptiveSparkPlan (107) +AdaptiveSparkPlan (109) +- == Final Plan == - VeloxColumnarToRowExec (70) - +- ^ SortExecTransformer (68) - +- ^ InputIteratorTransformer (67) - +- ShuffleQueryStage (65) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- AQEShuffleRead (7) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- BroadcastQueryStage (48) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ FilterExecTransformer (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- BroadcastQueryStage (17) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ FilterExecTransformer (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ShuffleQueryStage (38) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ FilterExecTransformer (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- BroadcastQueryStage (30) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- BroadcastQueryStage (58) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ FilterExecTransformer (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (72) + +- ^ SortExecTransformer (70) + +- ^ InputIteratorTransformer (69) + +- ShuffleQueryStage (67) + +- ColumnarExchange (66) + +- VeloxAppendBatches (65) + +- ^ ProjectExecTransformer (63) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (62) + :- ^ ProjectExecTransformer (53) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (52) + : :- ^ InputIteratorTransformer (10) + : : +- AQEShuffleRead (8) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (51) + : +- BroadcastQueryStage (49) + : +- ColumnarBroadcastExchange (48) + : +- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (45) + : :- ^ InputIteratorTransformer (26) + : : +- BroadcastQueryStage (24) + : : +- ColumnarBroadcastExchange (23) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) + : : :- ^ FilterExecTransformer (12) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (20) + : : +- BroadcastQueryStage (18) + : : +- ColumnarBroadcastExchange (17) + : : +- ^ ProjectExecTransformer (15) + : : +- ^ FilterExecTransformer (14) + : : +- ^ Scan parquet (13) + : +- ^ FilterExecTransformer (44) + : +- ^ ProjectExecTransformer (43) + : +- ^ RegularHashAggregateExecTransformer (42) + : +- ^ InputIteratorTransformer (41) + : +- ShuffleQueryStage (39) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ FilterExecTransformer (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (61) + +- BroadcastQueryStage (59) + +- ColumnarBroadcastExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ Scan parquet (54) +- == Initial Plan == - Sort (106) - +- Exchange (105) - +- Project (104) - +- BroadcastHashJoin Inner BuildRight (103) - :- Project (98) - : +- ShuffledHashJoin LeftSemi BuildRight (97) - : :- Exchange (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- Exchange (96) - : +- Project (95) - : +- BroadcastHashJoin Inner BuildLeft (94) - : :- BroadcastExchange (81) - : : +- BroadcastHashJoin LeftSemi BuildRight (80) - : : :- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (79) - : : +- Project (78) - : : +- Filter (77) - : : +- Scan parquet (76) - : +- Filter (93) - : +- HashAggregate (92) - : +- Exchange (91) - : +- HashAggregate (90) - : +- BroadcastHashJoin LeftSemi BuildRight (89) - : :- Project (84) - : : +- Filter (83) - : : +- Scan parquet (82) - : +- BroadcastExchange (88) - : +- Project (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (102) - +- Project (101) - +- Filter (100) - +- Scan parquet (99) + Sort (108) + +- Exchange (107) + +- Project (106) + +- BroadcastHashJoin Inner BuildRight (105) + :- Project (100) + : +- ShuffledHashJoin LeftSemi BuildRight (99) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Exchange (98) + : +- Project (97) + : +- BroadcastHashJoin Inner BuildLeft (96) + : :- BroadcastExchange (83) + : : +- BroadcastHashJoin LeftSemi BuildRight (82) + : : :- Filter (77) + : : : +- Scan parquet (76) + : : +- BroadcastExchange (81) + : : +- Project (80) + : : +- Filter (79) + : : +- Scan parquet (78) + : +- Filter (95) + : +- HashAggregate (94) + : +- Exchange (93) + : +- HashAggregate (92) + : +- BroadcastHashJoin LeftSemi BuildRight (91) + : :- Project (86) + : : +- Filter (85) + : : +- Scan parquet (84) + : +- BroadcastExchange (90) + : +- Project (89) + : +- Filter (88) + : +- Scan parquet (87) + +- BroadcastExchange (104) + +- Project (103) + +- Filter (102) + +- Scan parquet (101) (1) Scan parquet @@ -113,448 +115,456 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(8) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(9) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(11) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) FilterExecTransformer +(12) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(12) Scan parquet +(13) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) FilterExecTransformer +(14) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(14) ProjectExecTransformer +(15) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(18) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(27) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(30) ReusedExchange [Reuses operator id: 17] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(31) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(34) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(35) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(38) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(39) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(42) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(44) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(45) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(48) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(49) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(50) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(51) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(52) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(53) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(55) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(58) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(59) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(60) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(62) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(63) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(65) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(66) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_address#X] -(67) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(68) SortExecTransformer +(70) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(69) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(70) VeloxColumnarToRowExec +(72) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(71) Scan parquet +(73) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(72) Filter +(74) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(73) Exchange +(75) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) Scan parquet +(76) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(75) Filter +(77) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(76) Scan parquet +(78) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(77) Filter +(79) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(78) Project +(80) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(79) BroadcastExchange +(81) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(81) BroadcastExchange +(83) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) Scan parquet +(84) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(83) Filter +(85) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(84) Project +(86) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(85) Scan parquet +(87) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(86) Filter +(88) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(87) Project +(89) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(88) BroadcastExchange +(90) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) BroadcastHashJoin +(91) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(90) HashAggregate +(92) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(91) Exchange +(93) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) HashAggregate +(94) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Filter +(95) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(94) BroadcastHashJoin +(96) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(95) Project +(97) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(96) Exchange +(98) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(99) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(98) Project +(100) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(99) Scan parquet +(101) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(100) Filter +(102) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(101) Project +(103) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(102) BroadcastExchange +(104) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(103) BroadcastHashJoin +(105) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(104) Project +(106) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(105) Exchange +(107) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Sort +(108) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(107) AdaptiveSparkPlan +(109) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt index 4a970927299d9..d22e64b92b4ba 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/21.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (92) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (59) - +- TakeOrderedAndProjectExecTransformer (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (60) + +- TakeOrderedAndProjectExecTransformer (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (81) - : +- BroadcastHashJoin Inner BuildRight (80) - : :- Project (75) - : : +- BroadcastHashJoin Inner BuildLeft (74) - : : :- BroadcastExchange (62) - : : : +- Filter (61) - : : : +- Scan parquet (60) - : : +- BroadcastHashJoin LeftAnti BuildRight (73) - : : :- BroadcastHashJoin LeftSemi BuildRight (68) - : : : :- Project (65) - : : : : +- Filter (64) - : : : : +- Scan parquet (63) - : : : +- BroadcastExchange (67) - : : : +- Scan parquet (66) - : : +- BroadcastExchange (72) - : : +- Project (71) - : : +- Filter (70) - : : +- Scan parquet (69) - : +- BroadcastExchange (79) - : +- Project (78) - : +- Filter (77) - : +- Scan parquet (76) - +- BroadcastExchange (85) - +- Project (84) - +- Filter (83) - +- Scan parquet (82) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (82) + : +- BroadcastHashJoin Inner BuildRight (81) + : :- Project (76) + : : +- BroadcastHashJoin Inner BuildLeft (75) + : : :- BroadcastExchange (63) + : : : +- Filter (62) + : : : +- Scan parquet (61) + : : +- BroadcastHashJoin LeftAnti BuildRight (74) + : : :- BroadcastHashJoin LeftSemi BuildRight (69) + : : : :- Project (66) + : : : : +- Filter (65) + : : : : +- Scan parquet (64) + : : : +- BroadcastExchange (68) + : : : +- Scan parquet (67) + : : +- BroadcastExchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- BroadcastExchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- BroadcastExchange (86) + +- Project (85) + +- Filter (84) + +- Scan parquet (83) (1) Scan parquet @@ -286,7 +287,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] @@ -301,194 +302,198 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) TakeOrderedAndProjectExecTransformer +(59) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(59) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(60) Scan parquet +(61) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(61) Filter +(62) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(62) BroadcastExchange +(63) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(63) Scan parquet +(64) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(64) Filter +(65) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(65) Project +(66) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(67) BroadcastExchange +(68) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(68) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(69) Scan parquet +(70) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(70) Filter +(71) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(71) Project +(72) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(72) BroadcastExchange +(73) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(74) BroadcastHashJoin +(75) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(75) Project +(76) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(76) Scan parquet +(77) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(77) Filter +(78) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(78) Project +(79) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(79) BroadcastExchange +(80) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(80) BroadcastHashJoin +(81) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(81) Project +(82) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(82) Scan parquet +(83) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(83) Filter +(84) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(84) Project +(85) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(85) BroadcastExchange +(86) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(87) Project +(88) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(88) HashAggregate +(89) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(89) Exchange +(90) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(91) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(92) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt index d0252d7ac9976..9344c1bf2f70a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/22.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22) + +- ColumnarExchange (21) + +- VeloxAppendBatches (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (38) + +- Exchange (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- BroadcastHashJoin LeftAnti BuildRight (32) + :- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (31) + +- Scan parquet (30) (1) Scan parquet @@ -118,87 +119,91 @@ Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacc Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(20) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(21) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(25) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(28) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(29) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(30) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(31) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(32) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(33) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(34) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(37) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(38) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt index ba63341e7e03c..3cd44f2a745a1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ RegularHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -144,7 +145,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -159,131 +160,135 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt index 54c5c1b24d8bb..703f0f47e8825 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/4.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- BroadcastHashJoin LeftSemi BuildRight (38) + :- Project (33) + : +- Filter (32) + : +- Scan parquet (31) + +- BroadcastExchange (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -137,100 +138,104 @@ Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(31) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(32) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(33) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(36) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(37) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(39) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(40) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(41) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(43) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt index 2f037bd9c1d15..56a69c4dad354 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/5.txt @@ -1,91 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (101) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (100) + +- Exchange (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Project (72) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -337,196 +338,200 @@ Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedpric Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(59) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(60) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(69) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(74) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(79) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(84) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(85) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(87) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(89) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(92) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(95) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(96) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(97) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(98) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(98) Exchange +(99) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(100) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(101) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt index b39d8c1b2aec2..41f69e74d1a03 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt index 4b2aa1e644a2d..5e99678cabb75 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/7.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40) - +- ReusedExchange (39) + VeloxColumnarToRowExec (61) + +- ^ SortExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (83) + : +- BroadcastHashJoin Inner BuildRight (82) + : :- Project (78) + : : +- BroadcastHashJoin Inner BuildRight (77) + : : :- Project (73) + : : : +- BroadcastHashJoin Inner BuildRight (72) + : : : :- Project (68) + : : : : +- BroadcastHashJoin Inner BuildLeft (67) + : : : : :- BroadcastExchange (64) + : : : : : +- Filter (63) + : : : : : +- Scan parquet (62) + : : : : +- Filter (66) + : : : : +- Scan parquet (65) + : : : +- BroadcastExchange (71) + : : : +- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- BroadcastExchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- BroadcastExchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -307,188 +308,192 @@ Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(54) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(55) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(56) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(57) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(58) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(59) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(62) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(64) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(65) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(68) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(69) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(70) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(71) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(73) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(78) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(81) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(83) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(88) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(89) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(90) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(92) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(93) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt index fe9d785795989..f008d04b81aa2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/8.txt @@ -1,116 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (130) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ShuffleQueryStage (80) + +- ColumnarExchange (79) + +- VeloxAppendBatches (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (129) + +- Exchange (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- BroadcastHashJoin Inner BuildRight (123) + :- Project (118) + : +- BroadcastHashJoin Inner BuildRight (117) + : :- Project (113) + : : +- BroadcastHashJoin Inner BuildRight (112) + : : :- Project (108) + : : : +- BroadcastHashJoin Inner BuildRight (107) + : : : :- Project (103) + : : : : +- BroadcastHashJoin Inner BuildRight (102) + : : : : :- Project (98) + : : : : : +- BroadcastHashJoin Inner BuildRight (97) + : : : : : :- Project (93) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) + : : : : : : :- BroadcastExchange (89) + : : : : : : : +- Project (88) + : : : : : : : +- Filter (87) + : : : : : : : +- Scan parquet (86) + : : : : : : +- Filter (91) + : : : : : : +- Scan parquet (90) + : : : : : +- BroadcastExchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- BroadcastExchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- BroadcastExchange (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- BroadcastExchange (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- BroadcastExchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- BroadcastExchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -442,244 +443,248 @@ Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.000 Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(78) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(79) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(80) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(81) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(82) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(83) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(90) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(93) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(94) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(95) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(96) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(98) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(101) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(103) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(104) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(105) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(106) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(107) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(108) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(111) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(112) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(113) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(114) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(116) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(117) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(118) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(119) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(121) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(122) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(123) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(124) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(125) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(126) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(127) Exchange +(128) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(129) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(130) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt index cb62cd97d3e0e..dcfe2020f21c6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark32/9.txt @@ -1,89 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (98) + +- Exchange (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (69) + : : : : : +- Project (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -331,192 +332,196 @@ Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(59) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(63) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(73) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(78) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(80) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(83) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(88) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(91) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(93) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(94) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(97) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(98) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt index 89de3133895b1..2f3257bbdafc8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt index 898458b34cb7d..7ccdb0e045148 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -194,7 +195,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -209,155 +210,159 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt index 6677083c130dd..619379320e1c7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/11.txt @@ -1,55 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (58) + +- Exchange (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- BroadcastHashJoin Inner BuildRight (51) + :- Project (46) + : +- BroadcastHashJoin Inner BuildRight (45) + : :- Filter (41) + : : +- Scan parquet (40) + : +- BroadcastExchange (44) + : +- Filter (43) + : +- Scan parquet (42) + +- BroadcastExchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -187,350 +188,354 @@ Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery su Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(35) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(37) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(40) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(42) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(44) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(45) Project +(46) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(49) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(51) Project +(52) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(53) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(54) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(55) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(55) Filter +(56) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(57) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(58) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ FilterExecTransformer (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (83) + +- ^ ProjectExecTransformer (81) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ShuffleQueryStage (77), Statistics(X) + +- ColumnarExchange (76) + +- ^ FlushableHashAggregateExecTransformer (74) + +- ^ ProjectExecTransformer (73) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (72) + :- ^ ProjectExecTransformer (67) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + : :- ^ FilterExecTransformer (61) + : : +- ^ Scan parquet (60) + : +- ^ InputIteratorTransformer (65) + : +- BroadcastQueryStage (63), Statistics(X) + : +- ReusedExchange (62) + +- ^ InputIteratorTransformer (71) + +- BroadcastQueryStage (69), Statistics(X) + +- ReusedExchange (68) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Filter (85) + : : +- Scan parquet (84) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) + + +(60) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) FilterExecTransformer +(61) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(61) ReusedExchange [Reuses operator id: 6] +(62) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(63) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(64) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(65) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(66) ProjectExecTransformer +(67) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(68) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(69) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(70) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(71) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(72) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(72) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(74) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(76) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(77) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(78) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(79) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(80) ProjectExecTransformer +(81) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(81) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(84) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(86) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(89) Project +(90) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(93) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(95) Project +(96) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(97) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt index e83423c0b2f2d..efb7a9793b9e0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/12.txt @@ -1,41 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- VeloxAppendBatches (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (42) + +- Exchange (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- BroadcastHashJoin Inner BuildLeft (36) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -131,96 +132,100 @@ Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_ Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(27) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(31) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(32) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(33) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(34) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(35) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(36) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) Project +(37) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(38) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(39) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(41) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt index 1fa2d94b096b3..e75208be3e05e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/13.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- BroadcastHashJoin LeftOuter BuildRight (43) + :- Scan parquet (38) + +- BroadcastExchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -174,109 +175,113 @@ Results [2]: [c_count#X, count(1)#X AS custdist#X] Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(38) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(46) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(48) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(49) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(51) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt index 5bc1aef677905..332e9d086ddf5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -85,7 +86,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] @@ -96,95 +97,99 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt index 5cd45806c9fbc..e82d3da63e530 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/15.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- Project (39) + +- BroadcastHashJoin Inner BuildLeft (38) + :- BroadcastExchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -136,246 +137,250 @@ Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_ Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(36) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(38) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ FilterExecTransformer (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ FilterExecTransformer (44) + +- ^ Scan parquet (43) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(42) Scan parquet +(43) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) FilterExecTransformer +(44) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(46) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(49) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(53) ProjectExecTransformer +(54) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(59) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(60) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(61) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(62) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(63) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(64) HashAggregate +(65) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(66) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt index 56d39742c76e7..83440db8a3a52 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/16.txt @@ -1,53 +1,54 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- BroadcastHashJoin LeftAnti BuildRight (43) + : :- Filter (38) + : : +- Scan parquet (37) + : +- BroadcastExchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -175,134 +176,138 @@ Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_c Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(29) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(30) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(37) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(41) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(43) Scan parquet +(44) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(46) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(47) Project +(48) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(49) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(50) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(52) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(53) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(56) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt index 0b41400d5864d..a57c908f01784 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ RegularHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -47,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) FlushableHashAggregateExecTransformer +(5) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] @@ -58,141 +59,145 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt index c37f0b49567c9..a656d4e5433cc 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ RegularHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -123,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -138,333 +140,341 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt index 6822887d3aedb..8cc8b37b99095 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -84,7 +85,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -95,91 +96,95 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt index 32f28a0311c41..afee0913dbde0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/20.txt @@ -1,96 +1,98 @@ == Physical Plan == -AdaptiveSparkPlan (104) +AdaptiveSparkPlan (106) +- == Final Plan == - VeloxColumnarToRowExec (67) - +- AQEShuffleRead (66) - +- ShuffleQueryStage (65), Statistics(X) - +- ColumnarExchange (64) - +- ^ ProjectExecTransformer (62) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (61) - :- ^ ProjectExecTransformer (52) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (51) - : :- ^ InputIteratorTransformer (9) - : : +- AQEShuffleRead (7) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (50) - : +- BroadcastQueryStage (48), Statistics(X) - : +- ColumnarBroadcastExchange (47) - : +- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (44) - : :- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (20) - : : :- ^ FilterExecTransformer (11) - : : : +- ^ Scan parquet (10) - : : +- ^ InputIteratorTransformer (19) - : : +- BroadcastQueryStage (17), Statistics(X) - : : +- ColumnarBroadcastExchange (16) - : : +- ^ ProjectExecTransformer (14) - : : +- ^ FilterExecTransformer (13) - : : +- ^ Scan parquet (12) - : +- ^ FilterExecTransformer (43) - : +- ^ ProjectExecTransformer (42) - : +- ^ RegularHashAggregateExecTransformer (41) - : +- ^ InputIteratorTransformer (40) - : +- ShuffleQueryStage (38), Statistics(X) - : +- ColumnarExchange (37) - : +- ^ ProjectExecTransformer (35) - : +- ^ FlushableHashAggregateExecTransformer (34) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (33) - : :- ^ ProjectExecTransformer (28) - : : +- ^ FilterExecTransformer (27) - : : +- ^ Scan parquet (26) - : +- ^ InputIteratorTransformer (32) - : +- BroadcastQueryStage (30), Statistics(X) - : +- ReusedExchange (29) - +- ^ InputIteratorTransformer (60) - +- BroadcastQueryStage (58), Statistics(X) - +- ColumnarBroadcastExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ FilterExecTransformer (54) - +- ^ Scan parquet (53) + VeloxColumnarToRowExec (69) + +- AQEShuffleRead (68) + +- ShuffleQueryStage (67), Statistics(X) + +- ColumnarExchange (66) + +- VeloxAppendBatches (65) + +- ^ ProjectExecTransformer (63) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (62) + :- ^ ProjectExecTransformer (53) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (52) + : :- ^ InputIteratorTransformer (10) + : : +- AQEShuffleRead (8) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (51) + : +- BroadcastQueryStage (49), Statistics(X) + : +- ColumnarBroadcastExchange (48) + : +- ^ ProjectExecTransformer (46) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (45) + : :- ^ InputIteratorTransformer (26) + : : +- BroadcastQueryStage (24), Statistics(X) + : : +- ColumnarBroadcastExchange (23) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (21) + : : :- ^ FilterExecTransformer (12) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (20) + : : +- BroadcastQueryStage (18), Statistics(X) + : : +- ColumnarBroadcastExchange (17) + : : +- ^ ProjectExecTransformer (15) + : : +- ^ FilterExecTransformer (14) + : : +- ^ Scan parquet (13) + : +- ^ FilterExecTransformer (44) + : +- ^ ProjectExecTransformer (43) + : +- ^ RegularHashAggregateExecTransformer (42) + : +- ^ InputIteratorTransformer (41) + : +- ShuffleQueryStage (39), Statistics(X) + : +- ColumnarExchange (38) + : +- ^ ProjectExecTransformer (36) + : +- ^ FlushableHashAggregateExecTransformer (35) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (34) + : :- ^ ProjectExecTransformer (29) + : : +- ^ FilterExecTransformer (28) + : : +- ^ Scan parquet (27) + : +- ^ InputIteratorTransformer (33) + : +- BroadcastQueryStage (31), Statistics(X) + : +- ReusedExchange (30) + +- ^ InputIteratorTransformer (61) + +- BroadcastQueryStage (59), Statistics(X) + +- ColumnarBroadcastExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FilterExecTransformer (55) + +- ^ Scan parquet (54) +- == Initial Plan == - Sort (103) - +- Exchange (102) - +- Project (101) - +- BroadcastHashJoin Inner BuildRight (100) - :- Project (95) - : +- ShuffledHashJoin LeftSemi BuildRight (94) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Exchange (93) - : +- Project (92) - : +- BroadcastHashJoin Inner BuildLeft (91) - : :- BroadcastExchange (78) - : : +- BroadcastHashJoin LeftSemi BuildRight (77) - : : :- Filter (72) - : : : +- Scan parquet (71) - : : +- BroadcastExchange (76) - : : +- Project (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- Filter (90) - : +- HashAggregate (89) - : +- Exchange (88) - : +- HashAggregate (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Project (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (85) - : +- Project (84) - : +- Filter (83) - : +- Scan parquet (82) - +- BroadcastExchange (99) - +- Project (98) - +- Filter (97) - +- Scan parquet (96) + Sort (105) + +- Exchange (104) + +- Project (103) + +- BroadcastHashJoin Inner BuildRight (102) + :- Project (97) + : +- ShuffledHashJoin LeftSemi BuildRight (96) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (95) + : +- Project (94) + : +- BroadcastHashJoin Inner BuildLeft (93) + : :- BroadcastExchange (80) + : : +- BroadcastHashJoin LeftSemi BuildRight (79) + : : :- Filter (74) + : : : +- Scan parquet (73) + : : +- BroadcastExchange (78) + : : +- Project (77) + : : +- Filter (76) + : : +- Scan parquet (75) + : +- Filter (92) + : +- HashAggregate (91) + : +- Exchange (90) + : +- HashAggregate (89) + : +- BroadcastHashJoin LeftSemi BuildRight (88) + : :- Project (83) + : : +- Filter (82) + : : +- Scan parquet (81) + : +- BroadcastExchange (87) + : +- Project (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (101) + +- Project (100) + +- Filter (99) + +- Scan parquet (98) (1) Scan parquet @@ -112,438 +114,446 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) AQEShuffleRead +(8) AQEShuffleRead Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: local -(8) InputAdapter +(9) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(10) Scan parquet +(11) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(11) FilterExecTransformer +(12) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(12) Scan parquet +(13) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(13) FilterExecTransformer +(14) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(14) ProjectExecTransformer +(15) ProjectExecTransformer Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(15) WholeStageCodegenTransformer (X) +(16) WholeStageCodegenTransformer (X) Input [1]: [p_partkey#X] Arguments: false -(16) ColumnarBroadcastExchange +(17) ColumnarBroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(17) BroadcastQueryStage +(18) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(18) InputAdapter +(19) InputAdapter Input [1]: [p_partkey#X] -(19) InputIteratorTransformer +(20) InputIteratorTransformer Input [1]: [p_partkey#X] -(20) BroadcastHashJoinExecTransformer +(21) BroadcastHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(26) Scan parquet +(27) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(27) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(29) ReusedExchange [Reuses operator id: 16] +(30) ReusedExchange [Reuses operator id: 17] Output [1]: [p_partkey#X] -(30) BroadcastQueryStage +(31) BroadcastQueryStage Output [1]: [p_partkey#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [1]: [p_partkey#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [1]: [p_partkey#X] -(33) BroadcastHashJoinExecTransformer +(34) BroadcastHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(34) FlushableHashAggregateExecTransformer +(35) FlushableHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(36) ProjectExecTransformer Output [5]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(38) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(39) ShuffleQueryStage Output [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(42) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(43) FilterExecTransformer +(44) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(44) BroadcastHashJoinExecTransformer +(45) BroadcastHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(45) ProjectExecTransformer +(46) ProjectExecTransformer Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(46) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [1]: [ps_suppkey#X] Arguments: false -(47) ColumnarBroadcastExchange +(48) ColumnarBroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(48) BroadcastQueryStage +(49) BroadcastQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(49) InputAdapter +(50) InputAdapter Input [1]: [ps_suppkey#X] -(50) InputIteratorTransformer +(51) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(51) BroadcastHashJoinExecTransformer +(52) BroadcastHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(52) ProjectExecTransformer +(53) ProjectExecTransformer Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(53) Scan parquet +(54) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(54) FilterExecTransformer +(55) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(55) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [n_nationkey#X] Arguments: false -(57) ColumnarBroadcastExchange +(58) ColumnarBroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(58) BroadcastQueryStage +(59) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(59) InputAdapter +(60) InputAdapter Input [1]: [n_nationkey#X] -(60) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [n_nationkey#X] -(61) BroadcastHashJoinExecTransformer +(62) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(62) ProjectExecTransformer +(63) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(64) ColumnarExchange +(65) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(66) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(65) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(66) AQEShuffleRead +(68) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(67) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(68) Scan parquet +(70) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(69) Filter +(71) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(70) Exchange +(72) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(73) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(72) Filter +(74) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(73) Scan parquet +(75) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(74) Filter +(76) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(75) Project +(77) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(76) BroadcastExchange +(78) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(77) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(78) BroadcastExchange +(80) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(79) Scan parquet +(81) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(80) Filter +(82) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(81) Project +(83) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(82) Scan parquet +(84) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(83) Filter +(85) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(84) Project +(86) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(85) BroadcastExchange +(87) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(87) HashAggregate +(89) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(88) Exchange +(90) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(91) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(90) Filter +(92) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(91) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(92) Project +(94) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(93) Exchange +(95) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(95) Project +(97) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(96) Scan parquet +(98) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(97) Filter +(99) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(98) Project +(100) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(99) BroadcastExchange +(101) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(100) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(101) Project +(103) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(102) Exchange +(104) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Sort +(105) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(104) AdaptiveSparkPlan +(106) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt index 252a06b11fa78..d34e59327a672 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/21.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -285,7 +286,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] @@ -300,190 +301,194 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(74) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(80) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(86) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt index 5a6f0dc8fe05f..bf6c22d0d8b88 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/22.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- VeloxAppendBatches (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (38) + +- Exchange (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- BroadcastHashJoin LeftAnti BuildRight (32) + :- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (31) + +- Scan parquet (30) (1) Scan parquet @@ -118,221 +119,225 @@ Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacc Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(20) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(21) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(25) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(28) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(29) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(30) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(31) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(32) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(32) Project +(33) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(34) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(37) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(38) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (51) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- Filter (53) + +- Scan parquet (52) -(39) Scan parquet +(40) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) FilterExecTransformer +(41) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(41) ProjectExecTransformer +(42) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(45) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(46) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(47) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(48) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(51) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(52) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(53) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(54) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(55) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(56) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(57) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (51) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) \ No newline at end of file + HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- Filter (53) + +- Scan parquet (52) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt index 722def52f8508..5d022b4a97092 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ RegularHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -144,7 +145,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -159,131 +160,135 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt index 0ca16a6174b7c..577a71bdeff71 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/4.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- BroadcastHashJoin LeftSemi BuildRight (38) + :- Project (33) + : +- Filter (32) + : +- Scan parquet (31) + +- BroadcastExchange (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -137,100 +138,104 @@ Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(31) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(32) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(33) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(36) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(37) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(38) Project +(39) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(40) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(41) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(43) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt index 24ba48c495f6d..189b67f8e3531 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/5.txt @@ -1,91 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (101) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (100) + +- Exchange (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Project (72) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -337,196 +338,200 @@ Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedpric Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(59) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(60) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(69) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(73) Project +(74) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Project +(79) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(83) Project +(84) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(85) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(87) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(88) Project +(89) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(92) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(94) Project +(95) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(96) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(97) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(98) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(98) Exchange +(99) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(100) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(101) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt index 68854bdea4737..3a3148c1c1133 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt index fd1e1e8fa37fe..88226758f1035 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/7.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (61) + +- ^ SortExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (83) + : +- BroadcastHashJoin Inner BuildRight (82) + : :- Project (78) + : : +- BroadcastHashJoin Inner BuildRight (77) + : : :- Project (73) + : : : +- BroadcastHashJoin Inner BuildRight (72) + : : : :- Project (68) + : : : : +- BroadcastHashJoin Inner BuildLeft (67) + : : : : :- BroadcastExchange (64) + : : : : : +- Filter (63) + : : : : : +- Scan parquet (62) + : : : : +- Filter (66) + : : : : +- Scan parquet (65) + : : : +- BroadcastExchange (71) + : : : +- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- BroadcastExchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- BroadcastExchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -307,188 +308,192 @@ Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(54) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(55) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(56) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(57) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(58) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(59) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(62) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(64) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(65) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(67) Project +(68) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(69) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(70) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(71) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(72) Project +(73) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(77) Project +(78) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(81) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(83) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(88) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(89) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(90) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(92) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(93) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt index fc7a79a5b2b31..e0c8a43893bd6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/8.txt @@ -1,116 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (130) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ShuffleQueryStage (80), Statistics(X) + +- ColumnarExchange (79) + +- VeloxAppendBatches (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (129) + +- Exchange (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- BroadcastHashJoin Inner BuildRight (123) + :- Project (118) + : +- BroadcastHashJoin Inner BuildRight (117) + : :- Project (113) + : : +- BroadcastHashJoin Inner BuildRight (112) + : : :- Project (108) + : : : +- BroadcastHashJoin Inner BuildRight (107) + : : : :- Project (103) + : : : : +- BroadcastHashJoin Inner BuildRight (102) + : : : : :- Project (98) + : : : : : +- BroadcastHashJoin Inner BuildRight (97) + : : : : : :- Project (93) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) + : : : : : : :- BroadcastExchange (89) + : : : : : : : +- Project (88) + : : : : : : : +- Filter (87) + : : : : : : : +- Scan parquet (86) + : : : : : : +- Filter (91) + : : : : : : +- Scan parquet (90) + : : : : : +- BroadcastExchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- BroadcastExchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- BroadcastExchange (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- BroadcastExchange (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- BroadcastExchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- BroadcastExchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -442,244 +443,248 @@ Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.000 Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(78) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(79) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(80) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(81) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(82) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(83) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(90) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(92) Project +(93) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(94) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(95) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(96) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(97) Project +(98) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(101) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(102) Project +(103) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(104) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(105) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(106) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(107) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(107) Project +(108) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(111) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(112) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(112) Project +(113) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(114) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(116) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(117) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(118) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(119) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(121) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(122) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(123) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(123) Project +(124) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(125) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(126) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(127) Exchange +(128) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(129) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(130) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt index 4f18dfa352613..75b883057963b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark33/9.txt @@ -1,89 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (98) + +- Exchange (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (69) + : : : : : +- Project (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -331,192 +332,196 @@ Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(59) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(63) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(72) Project +(73) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(77) Project +(78) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(80) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(82) Project +(83) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(87) Project +(88) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(91) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(92) Project +(93) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(94) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(97) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(98) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt index 090a9522f13a8..468ad7c5d3aab 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt index 808f58b58fa5c..ce956550a45aa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/10.txt @@ -1,64 +1,65 @@ == Physical Plan == -AdaptiveSparkPlan (67) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (43) - +- TakeOrderedAndProjectExecTransformer (42) - +- ^ ProjectExecTransformer (40) - +- ^ RegularHashAggregateExecTransformer (39) - +- ^ InputIteratorTransformer (38) - +- ShuffleQueryStage (36), Statistics(X) - +- ColumnarExchange (35) - +- ^ ProjectExecTransformer (33) - +- ^ FlushableHashAggregateExecTransformer (32) - +- ^ ProjectExecTransformer (31) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) - :- ^ ProjectExecTransformer (22) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - : :- ^ ProjectExecTransformer (12) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - : : :- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (10) - : : +- BroadcastQueryStage (8), Statistics(X) - : : +- ColumnarBroadcastExchange (7) - : : +- ^ ProjectExecTransformer (5) - : : +- ^ FilterExecTransformer (4) - : : +- ^ Scan parquet (3) - : +- ^ InputIteratorTransformer (20) - : +- BroadcastQueryStage (18), Statistics(X) - : +- ColumnarBroadcastExchange (17) - : +- ^ ProjectExecTransformer (15) - : +- ^ FilterExecTransformer (14) - : +- ^ Scan parquet (13) - +- ^ InputIteratorTransformer (29) - +- BroadcastQueryStage (27), Statistics(X) - +- ColumnarBroadcastExchange (26) - +- ^ FilterExecTransformer (24) - +- ^ Scan parquet (23) + VeloxColumnarToRowExec (44) + +- TakeOrderedAndProjectExecTransformer (43) + +- ^ ProjectExecTransformer (41) + +- ^ RegularHashAggregateExecTransformer (40) + +- ^ InputIteratorTransformer (39) + +- ShuffleQueryStage (37), Statistics(X) + +- ColumnarExchange (36) + +- VeloxAppendBatches (35) + +- ^ ProjectExecTransformer (33) + +- ^ RegularHashAggregateExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (30) + :- ^ ProjectExecTransformer (22) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + : :- ^ ProjectExecTransformer (12) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + : : :- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (10) + : : +- BroadcastQueryStage (8), Statistics(X) + : : +- ColumnarBroadcastExchange (7) + : : +- ^ ProjectExecTransformer (5) + : : +- ^ FilterExecTransformer (4) + : : +- ^ Scan parquet (3) + : +- ^ InputIteratorTransformer (20) + : +- BroadcastQueryStage (18), Statistics(X) + : +- ColumnarBroadcastExchange (17) + : +- ^ ProjectExecTransformer (15) + : +- ^ FilterExecTransformer (14) + : +- ^ Scan parquet (13) + +- ^ InputIteratorTransformer (29) + +- BroadcastQueryStage (27), Statistics(X) + +- ColumnarBroadcastExchange (26) + +- ^ FilterExecTransformer (24) + +- ^ Scan parquet (23) +- == Initial Plan == - TakeOrderedAndProject (66) - +- HashAggregate (65) - +- Exchange (64) - +- HashAggregate (63) - +- Project (62) - +- BroadcastHashJoin Inner BuildRight (61) - :- Project (57) - : +- BroadcastHashJoin Inner BuildRight (56) - : :- Project (51) - : : +- BroadcastHashJoin Inner BuildRight (50) - : : :- Filter (45) - : : : +- Scan parquet (44) - : : +- BroadcastExchange (49) - : : +- Project (48) - : : +- Filter (47) - : : +- Scan parquet (46) - : +- BroadcastExchange (55) - : +- Project (54) - : +- Filter (53) - : +- Scan parquet (52) - +- BroadcastExchange (60) - +- Filter (59) - +- Scan parquet (58) + TakeOrderedAndProject (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- BroadcastHashJoin Inner BuildRight (62) + :- Project (58) + : +- BroadcastHashJoin Inner BuildRight (57) + : :- Project (52) + : : +- BroadcastHashJoin Inner BuildRight (51) + : : :- Filter (46) + : : : +- Scan parquet (45) + : : +- BroadcastExchange (50) + : : +- Project (49) + : : +- Filter (48) + : : +- Scan parquet (47) + : +- BroadcastExchange (56) + : +- Project (55) + : +- Filter (54) + : +- Scan parquet (53) + +- BroadcastExchange (61) + +- Filter (60) + +- Scan parquet (59) (1) Scan parquet @@ -197,7 +198,7 @@ Join condition: None Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(32) FlushableHashAggregateExecTransformer +(32) RegularHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -212,158 +213,162 @@ Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(35) ColumnarExchange +(35) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] +Arguments: X + +(36) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(37) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(37) InputAdapter +(38) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(38) InputIteratorTransformer +(39) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(39) RegularHashAggregateExecTransformer +(40) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(40) ProjectExecTransformer +(41) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(41) WholeStageCodegenTransformer (X) +(42) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(42) TakeOrderedAndProjectExecTransformer +(43) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(43) VeloxColumnarToRowExec +(44) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(44) Scan parquet +(45) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(45) Filter +(46) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(46) Scan parquet +(47) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(48) Project +(49) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(51) Project +(52) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(52) Scan parquet +(53) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(53) Filter +(54) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(54) Project +(55) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(55) BroadcastExchange +(56) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(56) BroadcastHashJoin +(57) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(57) Project +(58) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(58) Scan parquet +(59) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(59) Filter +(60) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(60) BroadcastExchange +(61) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(61) BroadcastHashJoin +(62) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(62) Project +(63) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(63) HashAggregate +(64) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(64) Exchange +(65) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(65) HashAggregate +(66) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(66) TakeOrderedAndProject +(67) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(67) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt index e6f8232f12c46..29763329fd30a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/11.txt @@ -1,55 +1,56 @@ == Physical Plan == -AdaptiveSparkPlan (58) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (38) - +- ^ SortExecTransformer (36) - +- ^ InputIteratorTransformer (35) - +- ShuffleQueryStage (33), Statistics(X) - +- ColumnarExchange (32) - +- ^ FilterExecTransformer (30) - +- ^ RegularHashAggregateExecTransformer (29) - +- ^ InputIteratorTransformer (28) - +- ShuffleQueryStage (26), Statistics(X) - +- ColumnarExchange (25) - +- ^ ProjectExecTransformer (23) - +- ^ FlushableHashAggregateExecTransformer (22) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - :- ^ ProjectExecTransformer (11) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (9) - : +- BroadcastQueryStage (7), Statistics(X) - : +- ColumnarBroadcastExchange (6) - : +- ^ FilterExecTransformer (4) - : +- ^ Scan parquet (3) - +- ^ InputIteratorTransformer (19) - +- BroadcastQueryStage (17), Statistics(X) - +- ColumnarBroadcastExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FilterExecTransformer (13) - +- ^ Scan parquet (12) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ FilterExecTransformer (30) + +- ^ RegularHashAggregateExecTransformer (29) + +- ^ InputIteratorTransformer (28) + +- ShuffleQueryStage (26), Statistics(X) + +- ColumnarExchange (25) + +- ^ ProjectExecTransformer (23) + +- ^ FlushableHashAggregateExecTransformer (22) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + :- ^ ProjectExecTransformer (11) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (9) + : +- BroadcastQueryStage (7), Statistics(X) + : +- ColumnarBroadcastExchange (6) + : +- ^ FilterExecTransformer (4) + : +- ^ Scan parquet (3) + +- ^ InputIteratorTransformer (19) + +- BroadcastQueryStage (17), Statistics(X) + +- ColumnarBroadcastExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FilterExecTransformer (13) + +- ^ Scan parquet (12) +- == Initial Plan == - Sort (57) - +- Exchange (56) - +- Filter (55) - +- HashAggregate (54) - +- Exchange (53) - +- HashAggregate (52) - +- Project (51) - +- BroadcastHashJoin Inner BuildRight (50) - :- Project (45) - : +- BroadcastHashJoin Inner BuildRight (44) - : :- Filter (40) - : : +- Scan parquet (39) - : +- BroadcastExchange (43) - : +- Filter (42) - : +- Scan parquet (41) - +- BroadcastExchange (49) - +- Project (48) - +- Filter (47) - +- Scan parquet (46) + Sort (58) + +- Exchange (57) + +- Filter (56) + +- HashAggregate (55) + +- Exchange (54) + +- HashAggregate (53) + +- Project (52) + +- BroadcastHashJoin Inner BuildRight (51) + :- Project (46) + : +- BroadcastHashJoin Inner BuildRight (45) + : :- Filter (41) + : : +- Scan parquet (40) + : +- BroadcastExchange (44) + : +- Filter (43) + : +- Scan parquet (42) + +- BroadcastExchange (50) + +- Project (49) + +- Filter (48) + +- Scan parquet (47) (1) Scan parquet @@ -189,356 +190,360 @@ Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery su Input [2]: [ps_partkey#X, value#X] Arguments: false -(32) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(33) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(34) InputAdapter +(35) InputAdapter Input [2]: [ps_partkey#X, value#X] -(35) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(36) SortExecTransformer +(37) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(37) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(38) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(39) Scan parquet +(40) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(41) Scan parquet +(42) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(42) Filter +(43) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(43) BroadcastExchange +(44) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(44) BroadcastHashJoin +(45) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(45) Project +(46) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(46) Scan parquet +(47) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(47) Filter +(48) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(48) Project +(49) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(49) BroadcastExchange +(50) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(50) BroadcastHashJoin +(51) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(51) Project +(52) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(52) HashAggregate +(53) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(53) Exchange +(54) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) HashAggregate +(55) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(55) Filter +(56) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(56) Exchange +(57) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) Sort +(58) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(58) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 30 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (99) +AdaptiveSparkPlan (100) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- ^ ProjectExecTransformer (80) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ FlushableHashAggregateExecTransformer (73) - +- ^ ProjectExecTransformer (72) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (71) - :- ^ ProjectExecTransformer (66) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (65) - : :- ^ FilterExecTransformer (60) - : : +- ^ Scan parquet (59) - : +- ^ InputIteratorTransformer (64) - : +- BroadcastQueryStage (62), Statistics(X) - : +- ReusedExchange (61) - +- ^ InputIteratorTransformer (70) - +- BroadcastQueryStage (68), Statistics(X) - +- ReusedExchange (67) + VeloxColumnarToRowExec (83) + +- ^ ProjectExecTransformer (81) + +- ^ RegularHashAggregateExecTransformer (80) + +- ^ InputIteratorTransformer (79) + +- ShuffleQueryStage (77), Statistics(X) + +- ColumnarExchange (76) + +- ^ FlushableHashAggregateExecTransformer (74) + +- ^ ProjectExecTransformer (73) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (72) + :- ^ ProjectExecTransformer (67) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + : :- ^ FilterExecTransformer (61) + : : +- ^ Scan parquet (60) + : +- ^ InputIteratorTransformer (65) + : +- BroadcastQueryStage (63), Statistics(X) + : +- ReusedExchange (62) + +- ^ InputIteratorTransformer (71) + +- BroadcastQueryStage (69), Statistics(X) + +- ReusedExchange (68) +- == Initial Plan == - HashAggregate (98) - +- Exchange (97) - +- HashAggregate (96) - +- Project (95) - +- BroadcastHashJoin Inner BuildRight (94) - :- Project (89) - : +- BroadcastHashJoin Inner BuildRight (88) - : :- Filter (84) - : : +- Scan parquet (83) - : +- BroadcastExchange (87) - : +- Filter (86) - : +- Scan parquet (85) - +- BroadcastExchange (93) - +- Project (92) - +- Filter (91) - +- Scan parquet (90) - - -(59) Scan parquet + HashAggregate (99) + +- Exchange (98) + +- HashAggregate (97) + +- Project (96) + +- BroadcastHashJoin Inner BuildRight (95) + :- Project (90) + : +- BroadcastHashJoin Inner BuildRight (89) + : :- Filter (85) + : : +- Scan parquet (84) + : +- BroadcastExchange (88) + : +- Filter (87) + : +- Scan parquet (86) + +- BroadcastExchange (94) + +- Project (93) + +- Filter (92) + +- Scan parquet (91) + + +(60) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(60) FilterExecTransformer +(61) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(61) ReusedExchange [Reuses operator id: 6] +(62) ReusedExchange [Reuses operator id: 6] Output [2]: [s_suppkey#X, s_nationkey#X] -(62) BroadcastQueryStage +(63) BroadcastQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(63) InputAdapter +(64) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(64) InputIteratorTransformer +(65) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(65) BroadcastHashJoinExecTransformer +(66) BroadcastHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(66) ProjectExecTransformer +(67) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(67) ReusedExchange [Reuses operator id: 16] +(68) ReusedExchange [Reuses operator id: 16] Output [1]: [n_nationkey#X] -(68) BroadcastQueryStage +(69) BroadcastQueryStage Output [1]: [n_nationkey#X] Arguments: X -(69) InputAdapter +(70) InputAdapter Input [1]: [n_nationkey#X] -(70) InputIteratorTransformer +(71) InputIteratorTransformer Input [1]: [n_nationkey#X] -(71) BroadcastHashJoinExecTransformer +(72) BroadcastHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(72) ProjectExecTransformer +(73) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(73) FlushableHashAggregateExecTransformer +(74) FlushableHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(74) WholeStageCodegenTransformer (X) +(75) WholeStageCodegenTransformer (X) Input [2]: [sum#X, isEmpty#X] Arguments: false -(75) ColumnarExchange +(76) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(77) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(77) InputAdapter +(78) InputAdapter Input [2]: [sum#X, isEmpty#X] -(78) InputIteratorTransformer +(79) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(79) RegularHashAggregateExecTransformer +(80) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(80) ProjectExecTransformer +(81) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(81) WholeStageCodegenTransformer (X) +(82) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(82) VeloxColumnarToRowExec +(83) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(83) Scan parquet +(84) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(85) Scan parquet +(86) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(87) BroadcastExchange +(88) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(88) BroadcastHashJoin +(89) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(89) Project +(90) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(90) Scan parquet +(91) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(91) Filter +(92) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(92) Project +(93) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(93) BroadcastExchange +(94) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(94) BroadcastHashJoin +(95) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(95) Project +(96) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(96) HashAggregate +(97) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(97) Exchange +(98) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(98) HashAggregate +(99) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(99) AdaptiveSparkPlan +(100) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt index 261e4fc821eac..8b6161e1f2241 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/12.txt @@ -1,41 +1,42 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (43) +- == Final Plan == - VeloxColumnarToRowExec (28) - +- ^ SortExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ InputIteratorTransformer (19) - +- ShuffleQueryStage (17), Statistics(X) - +- ColumnarExchange (16) - +- ^ ProjectExecTransformer (14) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (29) + +- ^ SortExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- VeloxAppendBatches (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- BroadcastHashJoin Inner BuildLeft (35) - :- BroadcastExchange (31) - : +- Filter (30) - : +- Scan parquet (29) - +- Project (34) - +- Filter (33) - +- Scan parquet (32) + Sort (42) + +- Exchange (41) + +- HashAggregate (40) + +- Exchange (39) + +- HashAggregate (38) + +- Project (37) + +- BroadcastHashJoin Inner BuildLeft (36) + :- BroadcastExchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -132,97 +133,101 @@ Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_ Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(22) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(26) SortExecTransformer +(27) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(27) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(30) Filter +(31) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(31) BroadcastExchange +(32) BroadcastExchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(32) Scan parquet +(33) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(33) Filter +(34) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(34) Project +(35) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(35) BroadcastHashJoin +(36) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(36) Project +(37) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(37) HashAggregate +(38) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(38) Exchange +(39) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(40) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(40) Exchange +(41) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(42) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(43) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt index 7ccf5dafe28ee..f84ec67ca1251 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/13.txt @@ -1,49 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) - :- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ ProjectExecTransformer (4) - +- ^ FilterExecTransformer (3) - +- ^ Scan parquet (2) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer LeftOuter BuildRight (10) + :- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ ProjectExecTransformer (4) + +- ^ FilterExecTransformer (3) + +- ^ Scan parquet (2) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- BroadcastHashJoin LeftOuter BuildRight (42) - :- Scan parquet (37) - +- BroadcastExchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- HashAggregate (47) + +- Exchange (46) + +- HashAggregate (45) + +- Project (44) + +- BroadcastHashJoin LeftOuter BuildRight (43) + :- Scan parquet (38) + +- BroadcastExchange (42) + +- Project (41) + +- Filter (40) + +- Scan parquet (39) (1) Scan parquet @@ -175,110 +176,114 @@ Results [2]: [c_count#X, count(1)#X AS custdist#X] Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(33) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(35) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(38) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Scan parquet +(39) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(39) Filter +(40) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(40) Project +(41) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(43) Project +(44) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(44) HashAggregate +(45) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(45) Exchange +(46) Exchange Input [2]: [c_custkey#X, count#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(47) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(48) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(49) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(50) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(51) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(52) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt index 8655ec75dd6f3..ef33284c5ef74 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/14.txt @@ -1,35 +1,36 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (35) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (23) + +- ^ ProjectExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (33) - +- Exchange (32) - +- HashAggregate (31) - +- Project (30) - +- BroadcastHashJoin Inner BuildRight (29) - :- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- BroadcastExchange (28) - +- Filter (27) - +- Scan parquet (26) + HashAggregate (34) + +- Exchange (33) + +- HashAggregate (32) + +- Project (31) + +- BroadcastHashJoin Inner BuildRight (30) + :- Project (26) + : +- Filter (25) + : +- Scan parquet (24) + +- BroadcastExchange (29) + +- Filter (28) + +- Scan parquet (27) (1) Scan parquet @@ -86,7 +87,7 @@ Join condition: None Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] @@ -97,96 +98,100 @@ Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(23) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(23) Scan parquet +(24) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(25) Project +(26) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) Scan parquet +(27) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(27) Filter +(28) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(28) BroadcastExchange +(29) BroadcastExchange Input [2]: [p_partkey#X, p_type#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(29) BroadcastHashJoin +(30) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(30) Project +(31) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(31) HashAggregate +(32) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(32) Exchange +(33) Exchange Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(33) HashAggregate +(34) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(34) AdaptiveSparkPlan +(35) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt index 0853cd6948f96..978722c255d7d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/15.txt @@ -1,42 +1,43 @@ == Physical Plan == -AdaptiveSparkPlan (41) +AdaptiveSparkPlan (42) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- AQEShuffleRead (25) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) - :- ^ InputIteratorTransformer (7) - : +- BroadcastQueryStage (5), Statistics(X) - : +- ColumnarBroadcastExchange (4) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (27) + +- AQEShuffleRead (26) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ ProjectExecTransformer (21) + +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (20) + :- ^ InputIteratorTransformer (7) + : +- BroadcastQueryStage (5), Statistics(X) + : +- ColumnarBroadcastExchange (4) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (19) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ FilterExecTransformer (9) + +- ^ Scan parquet (8) +- == Initial Plan == - Sort (40) - +- Exchange (39) - +- Project (38) - +- BroadcastHashJoin Inner BuildLeft (37) - :- BroadcastExchange (29) - : +- Filter (28) - : +- Scan parquet (27) - +- Filter (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- Filter (31) - +- Scan parquet (30) + Sort (41) + +- Exchange (40) + +- Project (39) + +- BroadcastHashJoin Inner BuildLeft (38) + :- BroadcastExchange (30) + : +- Filter (29) + : +- Scan parquet (28) + +- Filter (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- Filter (32) + +- Scan parquet (31) (1) Scan parquet @@ -137,247 +138,251 @@ Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_ Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(24) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(25) AQEShuffleRead +(26) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) Scan parquet +(28) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(28) Filter +(29) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(30) Scan parquet +(31) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(31) Filter +(32) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(32) Project +(33) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(33) HashAggregate +(34) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(36) Filter +(37) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(38) Project +(39) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(39) Exchange +(40) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(40) Sort +(41) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(41) AdaptiveSparkPlan +(42) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 19 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (66) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (57) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ ProjectExecTransformer (53) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ FilterExecTransformer (43) - +- ^ Scan parquet (42) + VeloxColumnarToRowExec (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- ^ ProjectExecTransformer (47) + +- ^ FlushableHashAggregateExecTransformer (46) + +- ^ ProjectExecTransformer (45) + +- ^ FilterExecTransformer (44) + +- ^ Scan parquet (43) +- == Initial Plan == - HashAggregate (65) - +- HashAggregate (64) - +- HashAggregate (63) - +- Exchange (62) - +- HashAggregate (61) - +- Project (60) - +- Filter (59) - +- Scan parquet (58) + HashAggregate (66) + +- HashAggregate (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- Project (61) + +- Filter (60) + +- Scan parquet (59) -(42) Scan parquet +(43) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(43) FilterExecTransformer +(44) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) FlushableHashAggregateExecTransformer +(46) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(46) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) WholeStageCodegenTransformer (X) +(48) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(48) ColumnarExchange +(49) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(49) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(50) InputAdapter +(51) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(51) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) RegularHashAggregateExecTransformer +(53) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(53) ProjectExecTransformer +(54) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(55) RegularHashAggregateExecTransformer +(56) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(56) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(57) VeloxColumnarToRowExec +(58) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(58) Scan parquet +(59) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(59) Filter +(60) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(60) Project +(61) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(61) HashAggregate +(62) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(62) Exchange +(63) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(64) HashAggregate +(65) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(65) HashAggregate +(66) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(66) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt index 78c65e72e6c88..fe411808b790b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/16.txt @@ -1,53 +1,54 @@ == Physical Plan == -AdaptiveSparkPlan (56) +AdaptiveSparkPlan (57) +- == Final Plan == - VeloxColumnarToRowExec (35) - +- ^ SortExecTransformer (33) - +- ^ InputIteratorTransformer (32) - +- ShuffleQueryStage (30), Statistics(X) - +- ColumnarExchange (29) - +- ^ RegularHashAggregateExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ ProjectExecTransformer (21) - +- ^ FlushableHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (9) - +- BroadcastQueryStage (7), Statistics(X) - +- ColumnarBroadcastExchange (6) - +- ^ FilterExecTransformer (4) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ RegularHashAggregateExecTransformer (19) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- ^ ProjectExecTransformer (13) + +- ^ FlushableHashAggregateExecTransformer (12) + +- ^ ProjectExecTransformer (11) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (10) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (9) + +- BroadcastQueryStage (7), Statistics(X) + +- ColumnarBroadcastExchange (6) + +- ^ FilterExecTransformer (4) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (55) - +- Exchange (54) - +- HashAggregate (53) - +- Exchange (52) - +- HashAggregate (51) - +- HashAggregate (50) - +- Exchange (49) - +- HashAggregate (48) - +- Project (47) - +- BroadcastHashJoin Inner BuildRight (46) - :- BroadcastHashJoin LeftAnti BuildRight (42) - : :- Filter (37) - : : +- Scan parquet (36) - : +- BroadcastExchange (41) - : +- Project (40) - : +- Filter (39) - : +- Scan parquet (38) - +- BroadcastExchange (45) - +- Filter (44) - +- Scan parquet (43) + Sort (56) + +- Exchange (55) + +- HashAggregate (54) + +- Exchange (53) + +- HashAggregate (52) + +- HashAggregate (51) + +- Exchange (50) + +- HashAggregate (49) + +- Project (48) + +- BroadcastHashJoin Inner BuildRight (47) + :- BroadcastHashJoin LeftAnti BuildRight (43) + : :- Filter (38) + : : +- Scan parquet (37) + : +- BroadcastExchange (42) + : +- Project (41) + : +- Filter (40) + : +- Scan parquet (39) + +- BroadcastExchange (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -176,136 +177,140 @@ Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_c Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(29) ColumnarExchange +(29) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(30) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(31) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(31) InputAdapter +(32) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(32) InputIteratorTransformer +(33) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(33) SortExecTransformer +(34) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(34) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(35) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(36) Scan parquet +(37) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(37) Filter +(38) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(38) Scan parquet +(39) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(39) Filter +(40) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(40) Project +(41) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(41) BroadcastExchange +(42) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(42) BroadcastHashJoin +(43) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(43) Scan parquet +(44) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(45) BroadcastExchange +(46) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(46) BroadcastHashJoin +(47) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(47) Project +(48) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(48) HashAggregate +(49) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(49) Exchange +(50) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) HashAggregate +(51) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(51) HashAggregate +(52) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(52) Exchange +(53) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(53) HashAggregate +(54) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(54) Exchange +(55) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(55) Sort +(56) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(56) AdaptiveSparkPlan +(57) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt index 62612096db425..e499293269fc8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/17.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (14) - +- ^ ProjectExecTransformer (12) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ FlushableHashAggregateExecTransformer (5) - +- ^ InputIteratorTransformer (4) - +- RowToVeloxColumnar (2) - +- LocalTableScan (1) + VeloxColumnarToRowExec (15) + +- ^ ProjectExecTransformer (13) + +- ^ RegularHashAggregateExecTransformer (12) + +- ^ InputIteratorTransformer (11) + +- ShuffleQueryStage (9), Statistics(X) + +- ColumnarExchange (8) + +- VeloxAppendBatches (7) + +- ^ RegularHashAggregateExecTransformer (5) + +- ^ InputIteratorTransformer (4) + +- RowToVeloxColumnar (2) + +- LocalTableScan (1) +- == Initial Plan == - HashAggregate (34) - +- Exchange (33) - +- HashAggregate (32) - +- Project (31) - +- BroadcastHashJoin Inner BuildRight (30) - :- Project (22) - : +- BroadcastHashJoin Inner BuildRight (21) - : :- Filter (16) - : : +- Scan parquet (15) - : +- BroadcastExchange (20) - : +- Project (19) - : +- Filter (18) - : +- Scan parquet (17) - +- BroadcastExchange (29) - +- Filter (28) - +- HashAggregate (27) - +- Exchange (26) - +- HashAggregate (25) - +- Filter (24) - +- Scan parquet (23) + HashAggregate (35) + +- Exchange (34) + +- HashAggregate (33) + +- Project (32) + +- BroadcastHashJoin Inner BuildRight (31) + :- Project (23) + : +- BroadcastHashJoin Inner BuildRight (22) + : :- Filter (17) + : : +- Scan parquet (16) + : +- BroadcastExchange (21) + : +- Project (20) + : +- Filter (19) + : +- Scan parquet (18) + +- BroadcastExchange (30) + +- Filter (29) + +- HashAggregate (28) + +- Exchange (27) + +- HashAggregate (26) + +- Filter (25) + +- Scan parquet (24) (1) LocalTableScan @@ -47,7 +48,7 @@ Input [1]: [l_extendedprice#X] (4) InputIteratorTransformer Input [1]: [l_extendedprice#X] -(5) FlushableHashAggregateExecTransformer +(5) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] @@ -58,143 +59,147 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(7) ColumnarExchange +(7) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(8) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(8) ShuffleQueryStage +(9) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(9) InputAdapter +(10) InputAdapter Input [2]: [sum#X, isEmpty#X] -(10) InputIteratorTransformer +(11) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(11) RegularHashAggregateExecTransformer +(12) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(12) ProjectExecTransformer +(13) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(13) WholeStageCodegenTransformer (X) +(14) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(14) VeloxColumnarToRowExec +(15) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(15) Scan parquet +(16) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(16) Filter +(17) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(17) Scan parquet +(18) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(18) Filter +(19) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(19) Project +(20) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(20) BroadcastExchange +(21) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(21) BroadcastHashJoin +(22) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(22) Project +(23) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(23) Scan parquet +(24) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(24) Filter +(25) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(25) HashAggregate +(26) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(26) Exchange +(27) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) HashAggregate +(28) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(28) Filter +(29) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(29) BroadcastExchange +(30) BroadcastExchange Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=X] -(30) BroadcastHashJoin +(31) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(31) Project +(32) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(32) HashAggregate +(33) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) Exchange +(34) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(34) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(35) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt index 58751c9b885f9..8899b0c255aea 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/18.txt @@ -1,80 +1,82 @@ == Physical Plan == -AdaptiveSparkPlan (86) +AdaptiveSparkPlan (88) +- == Final Plan == - VeloxColumnarToRowExec (53) - +- TakeOrderedAndProjectExecTransformer (52) - +- ^ RegularHashAggregateExecTransformer (50) - +- ^ InputIteratorTransformer (49) - +- ShuffleQueryStage (47), Statistics(X) - +- ColumnarExchange (46) - +- ^ ProjectExecTransformer (44) - +- ^ FlushableHashAggregateExecTransformer (43) - +- ^ ProjectExecTransformer (42) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (41) - :- ^ ProjectExecTransformer (28) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : :- ^ InputIteratorTransformer (7) - : : +- BroadcastQueryStage (5), Statistics(X) - : : +- ColumnarBroadcastExchange (4) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ FilterExecTransformer (9) - : : +- ^ Scan parquet (8) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ColumnarBroadcastExchange (22) - : +- ^ ProjectExecTransformer (20) - : +- ^ FilterExecTransformer (19) - : +- ^ RegularHashAggregateExecTransformer (18) - : +- ^ InputIteratorTransformer (17) - : +- ShuffleQueryStage (15), Statistics(X) - : +- ColumnarExchange (14) - : +- ^ ProjectExecTransformer (12) - : +- ^ FlushableHashAggregateExecTransformer (11) - : +- ^ Scan parquet (10) - +- ^ InputIteratorTransformer (40) - +- BroadcastQueryStage (38), Statistics(X) - +- ColumnarBroadcastExchange (37) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (35) - :- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (34) - +- BroadcastQueryStage (32), Statistics(X) - +- ReusedExchange (31) + VeloxColumnarToRowExec (55) + +- TakeOrderedAndProjectExecTransformer (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- VeloxAppendBatches (47) + +- ^ ProjectExecTransformer (45) + +- ^ RegularHashAggregateExecTransformer (44) + +- ^ ProjectExecTransformer (43) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (42) + :- ^ ProjectExecTransformer (29) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (28) + : :- ^ InputIteratorTransformer (7) + : : +- BroadcastQueryStage (5), Statistics(X) + : : +- ColumnarBroadcastExchange (4) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (27) + : :- ^ FilterExecTransformer (9) + : : +- ^ Scan parquet (8) + : +- ^ InputIteratorTransformer (26) + : +- BroadcastQueryStage (24), Statistics(X) + : +- ColumnarBroadcastExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FilterExecTransformer (20) + : +- ^ RegularHashAggregateExecTransformer (19) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ RegularHashAggregateExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (41) + +- BroadcastQueryStage (39), Statistics(X) + +- ColumnarBroadcastExchange (38) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (36) + :- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (35) + +- BroadcastQueryStage (33), Statistics(X) + +- ReusedExchange (32) +- == Initial Plan == - TakeOrderedAndProject (85) - +- HashAggregate (84) - +- Exchange (83) - +- HashAggregate (82) - +- Project (81) - +- BroadcastHashJoin Inner BuildRight (80) - :- Project (68) - : +- BroadcastHashJoin Inner BuildLeft (67) - : :- BroadcastExchange (56) - : : +- Filter (55) - : : +- Scan parquet (54) - : +- BroadcastHashJoin LeftSemi BuildRight (66) - : :- Filter (58) - : : +- Scan parquet (57) - : +- BroadcastExchange (65) - : +- Project (64) - : +- Filter (63) - : +- HashAggregate (62) - : +- Exchange (61) - : +- HashAggregate (60) - : +- Scan parquet (59) - +- BroadcastExchange (79) - +- BroadcastHashJoin LeftSemi BuildRight (78) - :- Filter (70) - : +- Scan parquet (69) - +- BroadcastExchange (77) - +- Project (76) - +- Filter (75) - +- HashAggregate (74) - +- Exchange (73) - +- HashAggregate (72) - +- Scan parquet (71) + TakeOrderedAndProject (87) + +- HashAggregate (86) + +- Exchange (85) + +- HashAggregate (84) + +- Project (83) + +- BroadcastHashJoin Inner BuildRight (82) + :- Project (70) + : +- BroadcastHashJoin Inner BuildLeft (69) + : :- BroadcastExchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- BroadcastHashJoin LeftSemi BuildRight (68) + : :- Filter (60) + : : +- Scan parquet (59) + : +- BroadcastExchange (67) + : +- Project (66) + : +- Filter (65) + : +- HashAggregate (64) + : +- Exchange (63) + : +- HashAggregate (62) + : +- Scan parquet (61) + +- BroadcastExchange (81) + +- BroadcastHashJoin LeftSemi BuildRight (80) + :- Filter (72) + : +- Scan parquet (71) + +- BroadcastExchange (79) + +- Project (78) + +- Filter (77) + +- HashAggregate (76) + +- Exchange (75) + +- HashAggregate (74) + +- Scan parquet (73) (1) Scan parquet @@ -123,7 +125,7 @@ Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(11) FlushableHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] @@ -138,341 +140,349 @@ Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(14) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(15) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(16) InputAdapter +(17) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(17) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(18) RegularHashAggregateExecTransformer +(19) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(19) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(20) ProjectExecTransformer +(21) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(21) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [1]: [l_orderkey#X] Arguments: false -(22) ColumnarBroadcastExchange +(23) ColumnarBroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(23) BroadcastQueryStage +(24) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(24) InputAdapter +(25) InputAdapter Input [1]: [l_orderkey#X] -(25) InputIteratorTransformer +(26) InputIteratorTransformer Input [1]: [l_orderkey#X] -(26) BroadcastHashJoinExecTransformer +(27) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(27) BroadcastHashJoinExecTransformer +(28) BroadcastHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(28) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(29) Scan parquet +(30) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(30) FilterExecTransformer +(31) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(31) ReusedExchange [Reuses operator id: 22] +(32) ReusedExchange [Reuses operator id: 23] Output [1]: [l_orderkey#X] -(32) BroadcastQueryStage +(33) BroadcastQueryStage Output [1]: [l_orderkey#X] Arguments: X -(33) InputAdapter +(34) InputAdapter Input [1]: [l_orderkey#X] -(34) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [l_orderkey#X] -(35) BroadcastHashJoinExecTransformer +(36) BroadcastHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(36) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [2]: [l_orderkey#X, l_quantity#X] Arguments: false -(37) ColumnarBroadcastExchange +(38) ColumnarBroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(38) BroadcastQueryStage +(39) BroadcastQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(39) InputAdapter +(40) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(40) InputIteratorTransformer +(41) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(41) BroadcastHashJoinExecTransformer +(42) BroadcastHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(42) ProjectExecTransformer +(43) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(43) FlushableHashAggregateExecTransformer +(44) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(44) ProjectExecTransformer +(45) ProjectExecTransformer Output [8]: [hash(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 42) AS hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(45) WholeStageCodegenTransformer (X) +(46) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: false -(46) ColumnarExchange +(47) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] +Arguments: X + +(48) ColumnarExchange Input [8]: [hash_partition_key#X, c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(49) ShuffleQueryStage Output [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: X -(48) InputAdapter +(50) InputAdapter Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(49) InputIteratorTransformer +(51) InputIteratorTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(50) RegularHashAggregateExecTransformer +(52) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(51) WholeStageCodegenTransformer (X) +(53) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(52) TakeOrderedAndProjectExecTransformer +(54) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(53) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(54) Scan parquet +(56) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(55) Filter +(57) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(56) BroadcastExchange +(58) BroadcastExchange Input [2]: [c_custkey#X, c_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(57) Scan parquet +(59) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(58) Filter +(60) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(59) Scan parquet +(61) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(60) HashAggregate +(62) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(61) Exchange +(63) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(63) Filter +(65) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(64) Project +(66) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(65) BroadcastExchange +(67) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(66) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(67) BroadcastHashJoin +(69) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(68) Project +(70) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(69) Scan parquet +(71) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(72) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(71) Scan parquet +(73) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(74) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(75) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(76) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(77) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(78) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) BroadcastExchange +(79) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(79) BroadcastExchange +(81) BroadcastExchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(80) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(81) Project +(83) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(82) HashAggregate +(84) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(83) Exchange +(85) Exchange Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(84) HashAggregate +(86) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(85) TakeOrderedAndProject +(87) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(86) AdaptiveSparkPlan +(88) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt index e2f0c051b69a2..0373360159eb3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/19.txt @@ -1,34 +1,35 @@ == Physical Plan == -AdaptiveSparkPlan (33) +AdaptiveSparkPlan (34) +- == Final Plan == - VeloxColumnarToRowExec (21) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ FlushableHashAggregateExecTransformer (13) - +- ^ ProjectExecTransformer (12) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (10) - +- BroadcastQueryStage (8), Statistics(X) - +- ColumnarBroadcastExchange (7) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (22) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- VeloxAppendBatches (15) + +- ^ RegularHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (11) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (10) + +- BroadcastQueryStage (8), Statistics(X) + +- ColumnarBroadcastExchange (7) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - HashAggregate (32) - +- Exchange (31) - +- HashAggregate (30) - +- Project (29) - +- BroadcastHashJoin Inner BuildRight (28) - :- Project (24) - : +- Filter (23) - : +- Scan parquet (22) - +- BroadcastExchange (27) - +- Filter (26) - +- Scan parquet (25) + HashAggregate (33) + +- Exchange (32) + +- HashAggregate (31) + +- Project (30) + +- BroadcastHashJoin Inner BuildRight (29) + :- Project (25) + : +- Filter (24) + : +- Scan parquet (23) + +- BroadcastExchange (28) + +- Filter (27) + +- Scan parquet (26) (1) Scan parquet @@ -85,7 +86,7 @@ Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(13) FlushableHashAggregateExecTransformer +(13) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -96,92 +97,96 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(15) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(16) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [2]: [sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(20) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(21) VeloxColumnarToRowExec +(22) VeloxColumnarToRowExec Input [1]: [revenue#X] -(22) Scan parquet +(23) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(23) Filter +(24) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(24) Project +(25) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(25) Scan parquet +(26) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(26) Filter +(27) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(27) BroadcastExchange +(28) BroadcastExchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(28) BroadcastHashJoin +(29) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(29) Project +(30) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(30) HashAggregate +(31) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(31) Exchange +(32) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(32) HashAggregate +(33) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(33) AdaptiveSparkPlan +(34) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt index 3b7e5334a3ba4..8a1415a5bca2a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/20.txt @@ -1,90 +1,91 @@ == Physical Plan == -AdaptiveSparkPlan (96) +AdaptiveSparkPlan (97) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- AQEShuffleRead (59) - +- ShuffleQueryStage (58), Statistics(X) - +- ColumnarExchange (57) - +- ^ ProjectExecTransformer (55) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) - :- ^ ProjectExecTransformer (45) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (43) - : +- BroadcastQueryStage (41), Statistics(X) - : +- ColumnarBroadcastExchange (40) - : +- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) - : :- ^ InputIteratorTransformer (18) - : : +- BroadcastQueryStage (16), Statistics(X) - : : +- ColumnarBroadcastExchange (15) - : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) - : : :- ^ FilterExecTransformer (4) - : : : +- ^ Scan parquet (3) - : : +- ^ InputIteratorTransformer (12) - : : +- BroadcastQueryStage (10), Statistics(X) - : : +- ColumnarBroadcastExchange (9) - : : +- ^ ProjectExecTransformer (7) - : : +- ^ FilterExecTransformer (6) - : : +- ^ Scan parquet (5) - : +- ^ FilterExecTransformer (36) - : +- ^ ProjectExecTransformer (35) - : +- ^ RegularHashAggregateExecTransformer (34) - : +- ^ InputIteratorTransformer (33) - : +- ShuffleQueryStage (31), Statistics(X) - : +- ColumnarExchange (30) - : +- ^ ProjectExecTransformer (28) - : +- ^ FlushableHashAggregateExecTransformer (27) - : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) - : :- ^ ProjectExecTransformer (21) - : : +- ^ FilterExecTransformer (20) - : : +- ^ Scan parquet (19) - : +- ^ InputIteratorTransformer (25) - : +- BroadcastQueryStage (23), Statistics(X) - : +- ReusedExchange (22) - +- ^ InputIteratorTransformer (53) - +- BroadcastQueryStage (51), Statistics(X) - +- ColumnarBroadcastExchange (50) - +- ^ ProjectExecTransformer (48) - +- ^ FilterExecTransformer (47) - +- ^ Scan parquet (46) + VeloxColumnarToRowExec (61) + +- AQEShuffleRead (60) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- VeloxAppendBatches (57) + +- ^ ProjectExecTransformer (55) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (54) + :- ^ ProjectExecTransformer (45) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (44) + : :- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (43) + : +- BroadcastQueryStage (41), Statistics(X) + : +- ColumnarBroadcastExchange (40) + : +- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (37) + : :- ^ InputIteratorTransformer (18) + : : +- BroadcastQueryStage (16), Statistics(X) + : : +- ColumnarBroadcastExchange (15) + : : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (13) + : : :- ^ FilterExecTransformer (4) + : : : +- ^ Scan parquet (3) + : : +- ^ InputIteratorTransformer (12) + : : +- BroadcastQueryStage (10), Statistics(X) + : : +- ColumnarBroadcastExchange (9) + : : +- ^ ProjectExecTransformer (7) + : : +- ^ FilterExecTransformer (6) + : : +- ^ Scan parquet (5) + : +- ^ FilterExecTransformer (36) + : +- ^ ProjectExecTransformer (35) + : +- ^ RegularHashAggregateExecTransformer (34) + : +- ^ InputIteratorTransformer (33) + : +- ShuffleQueryStage (31), Statistics(X) + : +- ColumnarExchange (30) + : +- ^ ProjectExecTransformer (28) + : +- ^ FlushableHashAggregateExecTransformer (27) + : +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (26) + : :- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (25) + : +- BroadcastQueryStage (23), Statistics(X) + : +- ReusedExchange (22) + +- ^ InputIteratorTransformer (53) + +- BroadcastQueryStage (51), Statistics(X) + +- ColumnarBroadcastExchange (50) + +- ^ ProjectExecTransformer (48) + +- ^ FilterExecTransformer (47) + +- ^ Scan parquet (46) +- == Initial Plan == - Sort (95) - +- Exchange (94) - +- Project (93) - +- BroadcastHashJoin Inner BuildRight (92) - :- Project (87) - : +- BroadcastHashJoin LeftSemi BuildRight (86) - : :- Filter (62) - : : +- Scan parquet (61) - : +- BroadcastExchange (85) - : +- Project (84) - : +- BroadcastHashJoin Inner BuildLeft (83) - : :- BroadcastExchange (70) - : : +- BroadcastHashJoin LeftSemi BuildRight (69) - : : :- Filter (64) - : : : +- Scan parquet (63) - : : +- BroadcastExchange (68) - : : +- Project (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Filter (82) - : +- HashAggregate (81) - : +- Exchange (80) - : +- HashAggregate (79) - : +- BroadcastHashJoin LeftSemi BuildRight (78) - : :- Project (73) - : : +- Filter (72) - : : +- Scan parquet (71) - : +- BroadcastExchange (77) - : +- Project (76) - : +- Filter (75) - : +- Scan parquet (74) - +- BroadcastExchange (91) - +- Project (90) - +- Filter (89) - +- Scan parquet (88) + Sort (96) + +- Exchange (95) + +- Project (94) + +- BroadcastHashJoin Inner BuildRight (93) + :- Project (88) + : +- BroadcastHashJoin LeftSemi BuildRight (87) + : :- Filter (63) + : : +- Scan parquet (62) + : +- BroadcastExchange (86) + : +- Project (85) + : +- BroadcastHashJoin Inner BuildLeft (84) + : :- BroadcastExchange (71) + : : +- BroadcastHashJoin LeftSemi BuildRight (70) + : : :- Filter (65) + : : : +- Scan parquet (64) + : : +- BroadcastExchange (69) + : : +- Project (68) + : : +- Filter (67) + : : +- Scan parquet (66) + : +- Filter (83) + : +- HashAggregate (82) + : +- Exchange (81) + : +- HashAggregate (80) + : +- BroadcastHashJoin LeftSemi BuildRight (79) + : :- Project (74) + : : +- Filter (73) + : : +- Scan parquet (72) + : +- BroadcastExchange (78) + : +- Project (77) + : +- Filter (76) + : +- Scan parquet (75) + +- BroadcastExchange (92) + +- Project (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -329,195 +330,199 @@ Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] Input [2]: [s_name#X, s_address#X] Arguments: false -(57) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(58) ShuffleQueryStage +(59) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(59) AQEShuffleRead +(60) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(61) Scan parquet +(62) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(63) Scan parquet +(64) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(64) Filter +(65) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) BroadcastHashJoin +(70) BroadcastHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(70) BroadcastExchange +(71) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(71) Scan parquet +(72) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(72) Filter +(73) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(73) Project +(74) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(74) Scan parquet +(75) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(75) Filter +(76) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(76) Project +(77) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(77) BroadcastExchange +(78) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(78) BroadcastHashJoin +(79) BroadcastHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(79) HashAggregate +(80) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(80) Exchange +(81) Exchange Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) HashAggregate +(82) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(82) Filter +(83) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(83) BroadcastHashJoin +(84) BroadcastHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(84) Project +(85) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(85) BroadcastExchange +(86) BroadcastExchange Input [1]: [ps_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(87) Project +(88) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(90) Project +(91) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(91) BroadcastExchange +(92) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(92) BroadcastHashJoin +(93) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) Project +(94) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(94) Exchange +(95) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) Sort +(96) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(96) AdaptiveSparkPlan +(97) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt index f81832ea4c1db..48fc651f66f1f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/21.txt @@ -1,84 +1,85 @@ == Physical Plan == -AdaptiveSparkPlan (91) +AdaptiveSparkPlan (92) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (28) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) - : : :- ^ InputIteratorTransformer (7) - : : : +- BroadcastQueryStage (5), Statistics(X) - : : : +- ColumnarBroadcastExchange (4) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) - : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) - : : : :- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (16) - : : : +- BroadcastQueryStage (14), Statistics(X) - : : : +- ColumnarBroadcastExchange (13) - : : : +- ^ Scan parquet (11) - : : +- ^ InputIteratorTransformer (25) - : : +- BroadcastQueryStage (23), Statistics(X) - : : +- ColumnarBroadcastExchange (22) - : : +- ^ ProjectExecTransformer (20) - : : +- ^ FilterExecTransformer (19) - : : +- ^ Scan parquet (18) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ ProjectExecTransformer (31) - : +- ^ FilterExecTransformer (30) - : +- ^ Scan parquet (29) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- VeloxAppendBatches (52) + +- ^ ProjectExecTransformer (50) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (28) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (27) + : : :- ^ InputIteratorTransformer (7) + : : : +- BroadcastQueryStage (5), Statistics(X) + : : : +- ColumnarBroadcastExchange (4) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (26) + : : :- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (17) + : : : :- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (16) + : : : +- BroadcastQueryStage (14), Statistics(X) + : : : +- ColumnarBroadcastExchange (13) + : : : +- ^ Scan parquet (11) + : : +- ^ InputIteratorTransformer (25) + : : +- BroadcastQueryStage (23), Statistics(X) + : : +- ColumnarBroadcastExchange (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ FilterExecTransformer (19) + : : +- ^ Scan parquet (18) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ FilterExecTransformer (30) + : +- ^ Scan parquet (29) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ ProjectExecTransformer (41) + +- ^ FilterExecTransformer (40) + +- ^ Scan parquet (39) +- == Initial Plan == - TakeOrderedAndProject (90) - +- HashAggregate (89) - +- Exchange (88) - +- HashAggregate (87) - +- Project (86) - +- BroadcastHashJoin Inner BuildRight (85) - :- Project (80) - : +- BroadcastHashJoin Inner BuildRight (79) - : :- Project (74) - : : +- BroadcastHashJoin Inner BuildLeft (73) - : : :- BroadcastExchange (61) - : : : +- Filter (60) - : : : +- Scan parquet (59) - : : +- BroadcastHashJoin LeftAnti BuildRight (72) - : : :- BroadcastHashJoin LeftSemi BuildRight (67) - : : : :- Project (64) - : : : : +- Filter (63) - : : : : +- Scan parquet (62) - : : : +- BroadcastExchange (66) - : : : +- Scan parquet (65) - : : +- BroadcastExchange (71) - : : +- Project (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- BroadcastExchange (78) - : +- Project (77) - : +- Filter (76) - : +- Scan parquet (75) - +- BroadcastExchange (84) - +- Project (83) - +- Filter (82) - +- Scan parquet (81) + TakeOrderedAndProject (91) + +- HashAggregate (90) + +- Exchange (89) + +- HashAggregate (88) + +- Project (87) + +- BroadcastHashJoin Inner BuildRight (86) + :- Project (81) + : +- BroadcastHashJoin Inner BuildRight (80) + : :- Project (75) + : : +- BroadcastHashJoin Inner BuildLeft (74) + : : :- BroadcastExchange (62) + : : : +- Filter (61) + : : : +- Scan parquet (60) + : : +- BroadcastHashJoin LeftAnti BuildRight (73) + : : :- BroadcastHashJoin LeftSemi BuildRight (68) + : : : :- Project (65) + : : : : +- Filter (64) + : : : : +- Scan parquet (63) + : : : +- BroadcastExchange (67) + : : : +- Scan parquet (66) + : : +- BroadcastExchange (72) + : : +- Project (71) + : : +- Filter (70) + : : +- Scan parquet (69) + : +- BroadcastExchange (79) + : +- Project (78) + : +- Filter (77) + : +- Scan parquet (76) + +- BroadcastExchange (85) + +- Project (84) + +- Filter (83) + +- Scan parquet (82) (1) Scan parquet @@ -290,7 +291,7 @@ Join condition: None Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(49) FlushableHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] @@ -305,195 +306,199 @@ Input [2]: [s_name#X, count#X] Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(52) ColumnarExchange +(52) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, count#X] +Arguments: X + +(53) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(54) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(54) InputAdapter +(55) InputAdapter Input [2]: [s_name#X, count#X] -(55) InputIteratorTransformer +(56) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(56) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(57) WholeStageCodegenTransformer (X) +(58) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(58) VeloxColumnarToRowExec +(59) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(59) Scan parquet +(60) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(60) Filter +(61) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(61) BroadcastExchange +(62) BroadcastExchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(62) Scan parquet +(63) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(63) Filter +(64) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(64) Project +(65) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(66) BroadcastExchange +(67) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(67) BroadcastHashJoin +(68) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(68) Scan parquet +(69) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(69) Filter +(70) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(70) Project +(71) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(71) BroadcastExchange +(72) BroadcastExchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(73) BroadcastHashJoin +(74) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(74) Project +(75) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(75) Scan parquet +(76) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(76) Filter +(77) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(77) Project +(78) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(78) BroadcastExchange +(79) BroadcastExchange Input [1]: [o_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(79) BroadcastHashJoin +(80) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(80) Project +(81) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(81) Scan parquet +(82) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(82) Filter +(83) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(83) Project +(84) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(84) BroadcastExchange +(85) BroadcastExchange Input [1]: [n_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(85) BroadcastHashJoin +(86) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(86) Project +(87) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(87) HashAggregate +(88) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(88) Exchange +(89) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) HashAggregate +(90) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(90) TakeOrderedAndProject +(91) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(91) AdaptiveSparkPlan +(92) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt index 5dac875505eaa..b740990ca7803 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/22.txt @@ -1,37 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (38) +AdaptiveSparkPlan (39) +- == Final Plan == - VeloxColumnarToRowExec (26) - +- ^ SortExecTransformer (24) - +- ^ InputIteratorTransformer (23) - +- ShuffleQueryStage (21), Statistics(X) - +- ColumnarExchange (20) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ InputIteratorTransformer (17) - +- ShuffleQueryStage (15), Statistics(X) - +- ColumnarExchange (14) - +- ^ ProjectExecTransformer (12) - +- ^ FlushableHashAggregateExecTransformer (11) - +- ^ ProjectExecTransformer (10) - +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) - :- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (8) - +- BroadcastQueryStage (6), Statistics(X) - +- ColumnarBroadcastExchange (5) - +- ^ Scan parquet (3) + VeloxColumnarToRowExec (27) + +- ^ SortExecTransformer (25) + +- ^ InputIteratorTransformer (24) + +- ShuffleQueryStage (22), Statistics(X) + +- ColumnarExchange (21) + +- VeloxAppendBatches (20) + +- ^ RegularHashAggregateExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- ^ ProjectExecTransformer (12) + +- ^ FlushableHashAggregateExecTransformer (11) + +- ^ ProjectExecTransformer (10) + +- ^ BroadcastHashJoinExecTransformer LeftAnti BuildRight (9) + :- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (8) + +- BroadcastQueryStage (6), Statistics(X) + +- ColumnarBroadcastExchange (5) + +- ^ Scan parquet (3) +- == Initial Plan == - Sort (37) - +- Exchange (36) - +- HashAggregate (35) - +- Exchange (34) - +- HashAggregate (33) - +- Project (32) - +- BroadcastHashJoin LeftAnti BuildRight (31) - :- Filter (28) - : +- Scan parquet (27) - +- BroadcastExchange (30) - +- Scan parquet (29) + Sort (38) + +- Exchange (37) + +- HashAggregate (36) + +- Exchange (35) + +- HashAggregate (34) + +- Project (33) + +- BroadcastHashJoin LeftAnti BuildRight (32) + :- Filter (29) + : +- Scan parquet (28) + +- BroadcastExchange (31) + +- Scan parquet (30) (1) Scan parquet @@ -119,222 +120,226 @@ Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacc Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(20) ColumnarExchange +(20) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(21) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(22) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(22) InputAdapter +(23) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(23) InputIteratorTransformer +(24) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(24) SortExecTransformer +(25) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(25) WholeStageCodegenTransformer (X) +(26) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(26) VeloxColumnarToRowExec +(27) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(27) Scan parquet +(28) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(28) Filter +(29) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(29) Scan parquet +(30) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(30) BroadcastExchange +(31) BroadcastExchange Input [1]: [o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(31) BroadcastHashJoin +(32) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(32) Project +(33) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(33) HashAggregate +(34) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(34) Exchange +(35) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(35) HashAggregate +(36) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(36) Exchange +(37) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Sort +(38) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(38) AdaptiveSparkPlan +(39) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (51) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) + HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- Filter (53) + +- Scan parquet (52) -(39) Scan parquet +(40) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(40) FilterExecTransformer +(41) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(41) ProjectExecTransformer +(42) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(42) FlushableHashAggregateExecTransformer +(43) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(43) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(44) ColumnarExchange +(45) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(46) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(46) InputAdapter +(47) InputAdapter Input [2]: [sum#X, count#X] -(47) InputIteratorTransformer +(48) InputIteratorTransformer Input [2]: [sum#X, count#X] -(48) RegularHashAggregateExecTransformer +(49) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(49) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(50) VeloxColumnarToRowExec +(51) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(51) Scan parquet +(52) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(52) Filter +(53) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(53) Project +(54) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(54) HashAggregate +(55) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(55) Exchange +(56) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(56) HashAggregate +(57) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) AdaptiveSparkPlan +(58) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (58) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ RegularHashAggregateExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FlushableHashAggregateExecTransformer (42) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (51) + +- ^ RegularHashAggregateExecTransformer (49) + +- ^ InputIteratorTransformer (48) + +- ShuffleQueryStage (46), Statistics(X) + +- ColumnarExchange (45) + +- ^ FlushableHashAggregateExecTransformer (43) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - HashAggregate (56) - +- Exchange (55) - +- HashAggregate (54) - +- Project (53) - +- Filter (52) - +- Scan parquet (51) \ No newline at end of file + HashAggregate (57) + +- Exchange (56) + +- HashAggregate (55) + +- Project (54) + +- Filter (53) + +- Scan parquet (52) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt index 7ee3b1a0cfd4f..5eb8568582afe 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/3.txt @@ -1,52 +1,53 @@ == Physical Plan == -AdaptiveSparkPlan (53) +AdaptiveSparkPlan (54) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- TakeOrderedAndProjectExecTransformer (33) - +- ^ ProjectExecTransformer (31) - +- ^ RegularHashAggregateExecTransformer (30) - +- ^ InputIteratorTransformer (29) - +- ShuffleQueryStage (27), Statistics(X) - +- ColumnarExchange (26) - +- ^ ProjectExecTransformer (24) - +- ^ FlushableHashAggregateExecTransformer (23) - +- ^ ProjectExecTransformer (22) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) - :- ^ ProjectExecTransformer (12) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : :- ^ InputIteratorTransformer (8) - : : +- BroadcastQueryStage (6), Statistics(X) - : : +- ColumnarBroadcastExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (20) - +- BroadcastQueryStage (18), Statistics(X) - +- ColumnarBroadcastExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FilterExecTransformer (14) - +- ^ Scan parquet (13) + VeloxColumnarToRowExec (35) + +- TakeOrderedAndProjectExecTransformer (34) + +- ^ ProjectExecTransformer (32) + +- ^ RegularHashAggregateExecTransformer (31) + +- ^ InputIteratorTransformer (30) + +- ShuffleQueryStage (28), Statistics(X) + +- ColumnarExchange (27) + +- VeloxAppendBatches (26) + +- ^ ProjectExecTransformer (24) + +- ^ RegularHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (21) + :- ^ ProjectExecTransformer (12) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : :- ^ InputIteratorTransformer (8) + : : +- BroadcastQueryStage (6), Statistics(X) + : : +- ColumnarBroadcastExchange (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ FilterExecTransformer (10) + : +- ^ Scan parquet (9) + +- ^ InputIteratorTransformer (20) + +- BroadcastQueryStage (18), Statistics(X) + +- ColumnarBroadcastExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FilterExecTransformer (14) + +- ^ Scan parquet (13) +- == Initial Plan == - TakeOrderedAndProject (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Project (48) - +- BroadcastHashJoin Inner BuildRight (47) - :- Project (42) - : +- BroadcastHashJoin Inner BuildLeft (41) - : :- BroadcastExchange (38) - : : +- Project (37) - : : +- Filter (36) - : : +- Scan parquet (35) - : +- Filter (40) - : +- Scan parquet (39) - +- BroadcastExchange (46) - +- Project (45) - +- Filter (44) - +- Scan parquet (43) + TakeOrderedAndProject (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- Project (49) + +- BroadcastHashJoin Inner BuildRight (48) + :- Project (43) + : +- BroadcastHashJoin Inner BuildLeft (42) + : :- BroadcastExchange (39) + : : +- Project (38) + : : +- Filter (37) + : : +- Scan parquet (36) + : +- Filter (41) + : +- Scan parquet (40) + +- BroadcastExchange (47) + +- Project (46) + +- Filter (45) + +- Scan parquet (44) (1) Scan parquet @@ -146,7 +147,7 @@ Join condition: None Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(23) FlushableHashAggregateExecTransformer +(23) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] @@ -161,133 +162,137 @@ Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: false -(26) ColumnarExchange +(26) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] +Arguments: X + +(27) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(27) ShuffleQueryStage +(28) ShuffleQueryStage Output [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: X -(28) InputAdapter +(29) InputAdapter Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(29) InputIteratorTransformer +(30) InputIteratorTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(30) RegularHashAggregateExecTransformer +(31) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(31) ProjectExecTransformer +(32) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(32) WholeStageCodegenTransformer (X) +(33) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(33) TakeOrderedAndProjectExecTransformer +(34) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(34) VeloxColumnarToRowExec +(35) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(35) Scan parquet +(36) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(36) Filter +(37) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(37) Project +(38) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(38) BroadcastExchange +(39) BroadcastExchange Input [1]: [c_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(39) Scan parquet +(40) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(40) Filter +(41) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(41) BroadcastHashJoin +(42) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(42) Project +(43) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(43) Scan parquet +(44) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(44) Filter +(45) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(45) Project +(46) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) BroadcastExchange +(47) BroadcastExchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(47) BroadcastHashJoin +(48) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(48) Project +(49) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(49) HashAggregate +(50) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(50) Exchange +(51) Exchange Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, o_orderdate#X, o_shippriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(52) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(52) TakeOrderedAndProject +(53) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(53) AdaptiveSparkPlan +(54) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt index 65a268b411090..7c123f45309b9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/4.txt @@ -1,43 +1,44 @@ == Physical Plan == -AdaptiveSparkPlan (44) +AdaptiveSparkPlan (45) +- == Final Plan == - VeloxColumnarToRowExec (29) - +- ^ SortExecTransformer (27) - +- ^ InputIteratorTransformer (26) - +- ShuffleQueryStage (24), Statistics(X) - +- ColumnarExchange (23) - +- ^ RegularHashAggregateExecTransformer (21) - +- ^ InputIteratorTransformer (20) - +- ShuffleQueryStage (18), Statistics(X) - +- ColumnarExchange (17) - +- ^ ProjectExecTransformer (15) - +- ^ FlushableHashAggregateExecTransformer (14) - +- ^ ProjectExecTransformer (13) - +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) - :- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (11) - +- BroadcastQueryStage (9), Statistics(X) - +- ColumnarBroadcastExchange (8) - +- ^ ProjectExecTransformer (6) - +- ^ FilterExecTransformer (5) - +- ^ Scan parquet (4) + VeloxColumnarToRowExec (30) + +- ^ SortExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- VeloxAppendBatches (23) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ InputIteratorTransformer (20) + +- ShuffleQueryStage (18), Statistics(X) + +- ColumnarExchange (17) + +- ^ ProjectExecTransformer (15) + +- ^ FlushableHashAggregateExecTransformer (14) + +- ^ ProjectExecTransformer (13) + +- ^ BroadcastHashJoinExecTransformer LeftSemi BuildRight (12) + :- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (11) + +- BroadcastQueryStage (9), Statistics(X) + +- ColumnarBroadcastExchange (8) + +- ^ ProjectExecTransformer (6) + +- ^ FilterExecTransformer (5) + +- ^ Scan parquet (4) +- == Initial Plan == - Sort (43) - +- Exchange (42) - +- HashAggregate (41) - +- Exchange (40) - +- HashAggregate (39) - +- Project (38) - +- BroadcastHashJoin LeftSemi BuildRight (37) - :- Project (32) - : +- Filter (31) - : +- Scan parquet (30) - +- BroadcastExchange (36) - +- Project (35) - +- Filter (34) - +- Scan parquet (33) + Sort (44) + +- Exchange (43) + +- HashAggregate (42) + +- Exchange (41) + +- HashAggregate (40) + +- Project (39) + +- BroadcastHashJoin LeftSemi BuildRight (38) + :- Project (33) + : +- Filter (32) + : +- Scan parquet (31) + +- BroadcastExchange (37) + +- Project (36) + +- Filter (35) + +- Scan parquet (34) (1) Scan parquet @@ -138,101 +139,105 @@ Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(23) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(24) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(25) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(26) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(27) SortExecTransformer +(28) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(28) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(29) VeloxColumnarToRowExec +(30) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(30) Scan parquet +(31) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(31) Filter +(32) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(32) Project +(33) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(33) Scan parquet +(34) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(34) Filter +(35) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(35) Project +(36) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(36) BroadcastExchange +(37) BroadcastExchange Input [1]: [l_orderkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(37) BroadcastHashJoin +(38) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(38) Project +(39) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(39) HashAggregate +(40) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(40) Exchange +(41) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) HashAggregate +(42) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(42) Exchange +(43) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) Sort +(44) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(44) AdaptiveSparkPlan +(45) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt index a86ee299c7c33..d2b7ce10fb333 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/5.txt @@ -1,91 +1,92 @@ == Physical Plan == -AdaptiveSparkPlan (100) +AdaptiveSparkPlan (101) +- == Final Plan == - VeloxColumnarToRowExec (65) - +- ^ SortExecTransformer (63) - +- ^ InputIteratorTransformer (62) - +- ShuffleQueryStage (60), Statistics(X) - +- ColumnarExchange (59) - +- ^ RegularHashAggregateExecTransformer (57) - +- ^ InputIteratorTransformer (56) - +- ShuffleQueryStage (54), Statistics(X) - +- ColumnarExchange (53) - +- ^ ProjectExecTransformer (51) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ ProjectExecTransformer (10) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (47) - +- BroadcastQueryStage (45), Statistics(X) - +- ColumnarBroadcastExchange (44) - +- ^ ProjectExecTransformer (42) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (66) + +- ^ SortExecTransformer (64) + +- ^ InputIteratorTransformer (63) + +- ShuffleQueryStage (61), Statistics(X) + +- ColumnarExchange (60) + +- VeloxAppendBatches (59) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ InputIteratorTransformer (56) + +- ShuffleQueryStage (54), Statistics(X) + +- ColumnarExchange (53) + +- ^ ProjectExecTransformer (51) + +- ^ FlushableHashAggregateExecTransformer (50) + +- ^ ProjectExecTransformer (49) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (48) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ ProjectExecTransformer (10) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (47) + +- BroadcastQueryStage (45), Statistics(X) + +- ColumnarBroadcastExchange (44) + +- ^ ProjectExecTransformer (42) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (99) - +- Exchange (98) - +- HashAggregate (97) - +- Exchange (96) - +- HashAggregate (95) - +- Project (94) - +- BroadcastHashJoin Inner BuildRight (93) - :- Project (88) - : +- BroadcastHashJoin Inner BuildRight (87) - : :- Project (83) - : : +- BroadcastHashJoin Inner BuildRight (82) - : : :- Project (78) - : : : +- BroadcastHashJoin Inner BuildRight (77) - : : : :- Project (73) - : : : : +- BroadcastHashJoin Inner BuildLeft (72) - : : : : :- BroadcastExchange (68) - : : : : : +- Filter (67) - : : : : : +- Scan parquet (66) - : : : : +- Project (71) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (76) - : : : +- Filter (75) - : : : +- Scan parquet (74) - : : +- BroadcastExchange (81) - : : +- Filter (80) - : : +- Scan parquet (79) - : +- BroadcastExchange (86) - : +- Filter (85) - : +- Scan parquet (84) - +- BroadcastExchange (92) - +- Project (91) - +- Filter (90) - +- Scan parquet (89) + Sort (100) + +- Exchange (99) + +- HashAggregate (98) + +- Exchange (97) + +- HashAggregate (96) + +- Project (95) + +- BroadcastHashJoin Inner BuildRight (94) + :- Project (89) + : +- BroadcastHashJoin Inner BuildRight (88) + : :- Project (84) + : : +- BroadcastHashJoin Inner BuildRight (83) + : : :- Project (79) + : : : +- BroadcastHashJoin Inner BuildRight (78) + : : : :- Project (74) + : : : : +- BroadcastHashJoin Inner BuildLeft (73) + : : : : :- BroadcastExchange (69) + : : : : : +- Filter (68) + : : : : : +- Scan parquet (67) + : : : : +- Project (72) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (77) + : : : +- Filter (76) + : : : +- Scan parquet (75) + : : +- BroadcastExchange (82) + : : +- Filter (81) + : : +- Scan parquet (80) + : +- BroadcastExchange (87) + : +- Filter (86) + : +- Scan parquet (85) + +- BroadcastExchange (93) + +- Project (92) + +- Filter (91) + +- Scan parquet (90) (1) Scan parquet @@ -342,201 +343,205 @@ Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS reven Input [2]: [n_name#X, revenue#X] Arguments: false -(59) ColumnarExchange +(59) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(60) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(60) ShuffleQueryStage +(61) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(61) InputAdapter +(62) InputAdapter Input [2]: [n_name#X, revenue#X] -(62) InputIteratorTransformer +(63) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(63) SortExecTransformer +(64) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(64) WholeStageCodegenTransformer (X) +(65) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(65) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(66) Scan parquet +(67) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(67) Filter +(68) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(68) BroadcastExchange +(69) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(71) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(72) BroadcastHashJoin +(73) BroadcastHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(73) Project +(74) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(74) Scan parquet +(75) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(75) Filter +(76) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(76) BroadcastExchange +(77) BroadcastExchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(77) BroadcastHashJoin +(78) BroadcastHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(78) Project +(79) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(79) Scan parquet +(80) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(80) Filter +(81) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(81) BroadcastExchange +(82) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false], input[1, bigint, false]),false), [plan_id=X] -(82) BroadcastHashJoin +(83) BroadcastHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(83) Project +(84) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(84) Scan parquet +(85) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(85) Filter +(86) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(86) BroadcastExchange +(87) BroadcastExchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(87) BroadcastHashJoin +(88) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(88) Project +(89) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(89) Scan parquet +(90) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(91) Project +(92) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(92) BroadcastExchange +(93) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(93) BroadcastHashJoin +(94) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(94) Project +(95) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(95) HashAggregate +(96) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(96) Exchange +(97) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) HashAggregate +(98) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(98) Exchange +(99) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Sort +(100) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(100) AdaptiveSparkPlan +(101) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt index 12d6c3ea85e4e..415d85f46ee56 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt index 571b6c13b6fdf..311144d09ec91 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/7.txt @@ -1,85 +1,86 @@ == Physical Plan == -AdaptiveSparkPlan (93) +AdaptiveSparkPlan (94) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- ^ SortExecTransformer (58) - +- ^ InputIteratorTransformer (57) - +- ShuffleQueryStage (55), Statistics(X) - +- ColumnarExchange (54) - +- ^ RegularHashAggregateExecTransformer (52) - +- ^ InputIteratorTransformer (51) - +- ShuffleQueryStage (49), Statistics(X) - +- ColumnarExchange (48) - +- ^ ProjectExecTransformer (46) - +- ^ FlushableHashAggregateExecTransformer (45) - +- ^ ProjectExecTransformer (44) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) - :- ^ ProjectExecTransformer (38) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) - : :- ^ ProjectExecTransformer (29) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) - : : :- ^ ProjectExecTransformer (20) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) - : : : :- ^ ProjectExecTransformer (11) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) - : : : : :- ^ InputIteratorTransformer (7) - : : : : : +- BroadcastQueryStage (5), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (4) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (9) - : : : : +- ^ Scan parquet (8) - : : : +- ^ InputIteratorTransformer (18) - : : : +- BroadcastQueryStage (16), Statistics(X) - : : : +- ColumnarBroadcastExchange (15) - : : : +- ^ FilterExecTransformer (13) - : : : +- ^ Scan parquet (12) - : : +- ^ InputIteratorTransformer (27) - : : +- BroadcastQueryStage (25), Statistics(X) - : : +- ColumnarBroadcastExchange (24) - : : +- ^ FilterExecTransformer (22) - : : +- ^ Scan parquet (21) - : +- ^ InputIteratorTransformer (36) - : +- BroadcastQueryStage (34), Statistics(X) - : +- ColumnarBroadcastExchange (33) - : +- ^ FilterExecTransformer (31) - : +- ^ Scan parquet (30) - +- ^ InputIteratorTransformer (42) - +- BroadcastQueryStage (40), Statistics(X) - +- ReusedExchange (39) + VeloxColumnarToRowExec (61) + +- ^ SortExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- VeloxAppendBatches (54) + +- ^ RegularHashAggregateExecTransformer (52) + +- ^ InputIteratorTransformer (51) + +- ShuffleQueryStage (49), Statistics(X) + +- ColumnarExchange (48) + +- ^ ProjectExecTransformer (46) + +- ^ FlushableHashAggregateExecTransformer (45) + +- ^ ProjectExecTransformer (44) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (43) + :- ^ ProjectExecTransformer (38) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (37) + : :- ^ ProjectExecTransformer (29) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (28) + : : :- ^ ProjectExecTransformer (20) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (19) + : : : :- ^ ProjectExecTransformer (11) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (10) + : : : : :- ^ InputIteratorTransformer (7) + : : : : : +- BroadcastQueryStage (5), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (4) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (9) + : : : : +- ^ Scan parquet (8) + : : : +- ^ InputIteratorTransformer (18) + : : : +- BroadcastQueryStage (16), Statistics(X) + : : : +- ColumnarBroadcastExchange (15) + : : : +- ^ FilterExecTransformer (13) + : : : +- ^ Scan parquet (12) + : : +- ^ InputIteratorTransformer (27) + : : +- BroadcastQueryStage (25), Statistics(X) + : : +- ColumnarBroadcastExchange (24) + : : +- ^ FilterExecTransformer (22) + : : +- ^ Scan parquet (21) + : +- ^ InputIteratorTransformer (36) + : +- BroadcastQueryStage (34), Statistics(X) + : +- ColumnarBroadcastExchange (33) + : +- ^ FilterExecTransformer (31) + : +- ^ Scan parquet (30) + +- ^ InputIteratorTransformer (42) + +- BroadcastQueryStage (40), Statistics(X) + +- ReusedExchange (39) +- == Initial Plan == - Sort (92) - +- Exchange (91) - +- HashAggregate (90) - +- Exchange (89) - +- HashAggregate (88) - +- Project (87) - +- BroadcastHashJoin Inner BuildRight (86) - :- Project (82) - : +- BroadcastHashJoin Inner BuildRight (81) - : :- Project (77) - : : +- BroadcastHashJoin Inner BuildRight (76) - : : :- Project (72) - : : : +- BroadcastHashJoin Inner BuildRight (71) - : : : :- Project (67) - : : : : +- BroadcastHashJoin Inner BuildLeft (66) - : : : : :- BroadcastExchange (63) - : : : : : +- Filter (62) - : : : : : +- Scan parquet (61) - : : : : +- Filter (65) - : : : : +- Scan parquet (64) - : : : +- BroadcastExchange (70) - : : : +- Filter (69) - : : : +- Scan parquet (68) - : : +- BroadcastExchange (75) - : : +- Filter (74) - : : +- Scan parquet (73) - : +- BroadcastExchange (80) - : +- Filter (79) - : +- Scan parquet (78) - +- BroadcastExchange (85) - +- Filter (84) - +- Scan parquet (83) + Sort (93) + +- Exchange (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- BroadcastHashJoin Inner BuildRight (87) + :- Project (83) + : +- BroadcastHashJoin Inner BuildRight (82) + : :- Project (78) + : : +- BroadcastHashJoin Inner BuildRight (77) + : : :- Project (73) + : : : +- BroadcastHashJoin Inner BuildRight (72) + : : : :- Project (68) + : : : : +- BroadcastHashJoin Inner BuildLeft (67) + : : : : :- BroadcastExchange (64) + : : : : : +- Filter (63) + : : : : : +- Scan parquet (62) + : : : : +- Filter (66) + : : : : +- Scan parquet (65) + : : : +- BroadcastExchange (71) + : : : +- Filter (70) + : : : +- Scan parquet (69) + : : +- BroadcastExchange (76) + : : +- Filter (75) + : : +- Scan parquet (74) + : +- BroadcastExchange (81) + : +- Filter (80) + : +- Scan parquet (79) + +- BroadcastExchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -312,193 +313,197 @@ Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(54) ColumnarExchange +(54) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(55) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(55) ShuffleQueryStage +(56) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(56) InputAdapter +(57) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(57) InputIteratorTransformer +(58) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(58) SortExecTransformer +(59) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(59) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(60) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(61) Scan parquet +(62) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(62) Filter +(63) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(63) BroadcastExchange +(64) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(64) Scan parquet +(65) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(65) Filter +(66) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(66) BroadcastHashJoin +(67) BroadcastHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(67) Project +(68) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(68) Scan parquet +(69) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(69) Filter +(70) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(70) BroadcastExchange +(71) BroadcastExchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(72) Project +(73) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(77) Project +(78) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(79) Filter +(80) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(80) BroadcastExchange +(81) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(83) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(87) Project +(88) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(88) HashAggregate +(89) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(89) Exchange +(90) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) HashAggregate +(91) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(91) Exchange +(92) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) Sort +(93) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(93) AdaptiveSparkPlan +(94) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt index af98ad739c212..bcdebad75e778 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/8.txt @@ -1,116 +1,117 @@ == Physical Plan == -AdaptiveSparkPlan (129) +AdaptiveSparkPlan (130) +- == Final Plan == - VeloxColumnarToRowExec (84) - +- ^ SortExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ RegularHashAggregateExecTransformer (75) - +- ^ InputIteratorTransformer (74) - +- ShuffleQueryStage (72), Statistics(X) - +- ColumnarExchange (71) - +- ^ ProjectExecTransformer (69) - +- ^ FlushableHashAggregateExecTransformer (68) - +- ^ ProjectExecTransformer (67) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) - :- ^ ProjectExecTransformer (57) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) - : :- ^ ProjectExecTransformer (48) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ ProjectExecTransformer (39) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : : : :- ^ ProjectExecTransformer (30) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : : : :- ^ ProjectExecTransformer (21) - : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : : : :- ^ ProjectExecTransformer (12) - : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarBroadcastExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (19) - : : : : : +- BroadcastQueryStage (17), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (16) - : : : : : +- ^ FilterExecTransformer (14) - : : : : : +- ^ Scan parquet (13) - : : : : +- ^ InputIteratorTransformer (28) - : : : : +- BroadcastQueryStage (26), Statistics(X) - : : : : +- ColumnarBroadcastExchange (25) - : : : : +- ^ FilterExecTransformer (23) - : : : : +- ^ Scan parquet (22) - : : : +- ^ InputIteratorTransformer (37) - : : : +- BroadcastQueryStage (35), Statistics(X) - : : : +- ColumnarBroadcastExchange (34) - : : : +- ^ FilterExecTransformer (32) - : : : +- ^ Scan parquet (31) - : : +- ^ InputIteratorTransformer (46) - : : +- BroadcastQueryStage (44), Statistics(X) - : : +- ColumnarBroadcastExchange (43) - : : +- ^ FilterExecTransformer (41) - : : +- ^ Scan parquet (40) - : +- ^ InputIteratorTransformer (55) - : +- BroadcastQueryStage (53), Statistics(X) - : +- ColumnarBroadcastExchange (52) - : +- ^ FilterExecTransformer (50) - : +- ^ Scan parquet (49) - +- ^ InputIteratorTransformer (65) - +- BroadcastQueryStage (63), Statistics(X) - +- ColumnarBroadcastExchange (62) - +- ^ ProjectExecTransformer (60) - +- ^ FilterExecTransformer (59) - +- ^ Scan parquet (58) + VeloxColumnarToRowExec (85) + +- ^ SortExecTransformer (83) + +- ^ InputIteratorTransformer (82) + +- ShuffleQueryStage (80), Statistics(X) + +- ColumnarExchange (79) + +- VeloxAppendBatches (78) + +- ^ ProjectExecTransformer (76) + +- ^ RegularHashAggregateExecTransformer (75) + +- ^ InputIteratorTransformer (74) + +- ShuffleQueryStage (72), Statistics(X) + +- ColumnarExchange (71) + +- ^ ProjectExecTransformer (69) + +- ^ FlushableHashAggregateExecTransformer (68) + +- ^ ProjectExecTransformer (67) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (66) + :- ^ ProjectExecTransformer (57) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (56) + : :- ^ ProjectExecTransformer (48) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + : : :- ^ ProjectExecTransformer (39) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : : : :- ^ ProjectExecTransformer (30) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : : : :- ^ ProjectExecTransformer (21) + : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : : : :- ^ ProjectExecTransformer (12) + : : : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : : : :- ^ InputIteratorTransformer (8) + : : : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : : : +- ColumnarBroadcastExchange (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ FilterExecTransformer (10) + : : : : : : +- ^ Scan parquet (9) + : : : : : +- ^ InputIteratorTransformer (19) + : : : : : +- BroadcastQueryStage (17), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (16) + : : : : : +- ^ FilterExecTransformer (14) + : : : : : +- ^ Scan parquet (13) + : : : : +- ^ InputIteratorTransformer (28) + : : : : +- BroadcastQueryStage (26), Statistics(X) + : : : : +- ColumnarBroadcastExchange (25) + : : : : +- ^ FilterExecTransformer (23) + : : : : +- ^ Scan parquet (22) + : : : +- ^ InputIteratorTransformer (37) + : : : +- BroadcastQueryStage (35), Statistics(X) + : : : +- ColumnarBroadcastExchange (34) + : : : +- ^ FilterExecTransformer (32) + : : : +- ^ Scan parquet (31) + : : +- ^ InputIteratorTransformer (46) + : : +- BroadcastQueryStage (44), Statistics(X) + : : +- ColumnarBroadcastExchange (43) + : : +- ^ FilterExecTransformer (41) + : : +- ^ Scan parquet (40) + : +- ^ InputIteratorTransformer (55) + : +- BroadcastQueryStage (53), Statistics(X) + : +- ColumnarBroadcastExchange (52) + : +- ^ FilterExecTransformer (50) + : +- ^ Scan parquet (49) + +- ^ InputIteratorTransformer (65) + +- BroadcastQueryStage (63), Statistics(X) + +- ColumnarBroadcastExchange (62) + +- ^ ProjectExecTransformer (60) + +- ^ FilterExecTransformer (59) + +- ^ Scan parquet (58) +- == Initial Plan == - Sort (128) - +- Exchange (127) - +- HashAggregate (126) - +- Exchange (125) - +- HashAggregate (124) - +- Project (123) - +- BroadcastHashJoin Inner BuildRight (122) - :- Project (117) - : +- BroadcastHashJoin Inner BuildRight (116) - : :- Project (112) - : : +- BroadcastHashJoin Inner BuildRight (111) - : : :- Project (107) - : : : +- BroadcastHashJoin Inner BuildRight (106) - : : : :- Project (102) - : : : : +- BroadcastHashJoin Inner BuildRight (101) - : : : : :- Project (97) - : : : : : +- BroadcastHashJoin Inner BuildRight (96) - : : : : : :- Project (92) - : : : : : : +- BroadcastHashJoin Inner BuildLeft (91) - : : : : : : :- BroadcastExchange (88) - : : : : : : : +- Project (87) - : : : : : : : +- Filter (86) - : : : : : : : +- Scan parquet (85) - : : : : : : +- Filter (90) - : : : : : : +- Scan parquet (89) - : : : : : +- BroadcastExchange (95) - : : : : : +- Filter (94) - : : : : : +- Scan parquet (93) - : : : : +- BroadcastExchange (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- BroadcastExchange (105) - : : : +- Filter (104) - : : : +- Scan parquet (103) - : : +- BroadcastExchange (110) - : : +- Filter (109) - : : +- Scan parquet (108) - : +- BroadcastExchange (115) - : +- Filter (114) - : +- Scan parquet (113) - +- BroadcastExchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (129) + +- Exchange (128) + +- HashAggregate (127) + +- Exchange (126) + +- HashAggregate (125) + +- Project (124) + +- BroadcastHashJoin Inner BuildRight (123) + :- Project (118) + : +- BroadcastHashJoin Inner BuildRight (117) + : :- Project (113) + : : +- BroadcastHashJoin Inner BuildRight (112) + : : :- Project (108) + : : : +- BroadcastHashJoin Inner BuildRight (107) + : : : :- Project (103) + : : : : +- BroadcastHashJoin Inner BuildRight (102) + : : : : :- Project (98) + : : : : : +- BroadcastHashJoin Inner BuildRight (97) + : : : : : :- Project (93) + : : : : : : +- BroadcastHashJoin Inner BuildLeft (92) + : : : : : : :- BroadcastExchange (89) + : : : : : : : +- Project (88) + : : : : : : : +- Filter (87) + : : : : : : : +- Scan parquet (86) + : : : : : : +- Filter (91) + : : : : : : +- Scan parquet (90) + : : : : : +- BroadcastExchange (96) + : : : : : +- Filter (95) + : : : : : +- Scan parquet (94) + : : : : +- BroadcastExchange (101) + : : : : +- Filter (100) + : : : : +- Scan parquet (99) + : : : +- BroadcastExchange (106) + : : : +- Filter (105) + : : : +- Scan parquet (104) + : : +- BroadcastExchange (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- BroadcastExchange (116) + : +- Filter (115) + : +- Scan parquet (114) + +- BroadcastExchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) (1) Scan parquet @@ -449,251 +450,255 @@ Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.000 Input [2]: [o_year#X, mkt_share#X] Arguments: false -(78) ColumnarExchange +(78) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(79) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(80) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(80) InputAdapter +(81) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(81) InputIteratorTransformer +(82) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(82) SortExecTransformer +(83) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(83) WholeStageCodegenTransformer (X) +(84) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(84) VeloxColumnarToRowExec +(85) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(85) Scan parquet +(86) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(86) Filter +(87) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(87) Project +(88) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(88) BroadcastExchange +(89) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(89) Scan parquet +(90) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(90) Filter +(91) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(92) Project +(93) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(93) Scan parquet +(94) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(94) Filter +(95) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(95) BroadcastExchange +(96) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(96) BroadcastHashJoin +(97) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(97) Project +(98) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(98) Scan parquet +(99) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(99) Filter +(100) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(100) BroadcastExchange +(101) BroadcastExchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(101) BroadcastHashJoin +(102) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(102) Project +(103) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(103) Scan parquet +(104) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(104) Filter +(105) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(105) BroadcastExchange +(106) BroadcastExchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(106) BroadcastHashJoin +(107) BroadcastHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(107) Project +(108) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(108) Scan parquet +(109) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(109) Filter +(110) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(110) BroadcastExchange +(111) BroadcastExchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(111) BroadcastHashJoin +(112) BroadcastHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(112) Project +(113) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(113) Scan parquet +(114) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(114) Filter +(115) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(115) BroadcastExchange +(116) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(116) BroadcastHashJoin +(117) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(118) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(118) Scan parquet +(119) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(119) Filter +(120) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(120) Project +(121) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(121) BroadcastExchange +(122) BroadcastExchange Input [1]: [r_regionkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(122) BroadcastHashJoin +(123) BroadcastHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(123) Project +(124) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(124) HashAggregate +(125) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(125) Exchange +(126) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) HashAggregate +(127) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(127) Exchange +(128) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(128) Sort +(129) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(129) AdaptiveSparkPlan +(130) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt index c88eba3bb12e4..4a0f66b2e7462 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-bhj/spark34/9.txt @@ -1,89 +1,90 @@ == Physical Plan == -AdaptiveSparkPlan (98) +AdaptiveSparkPlan (99) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- ^ SortExecTransformer (62) - +- ^ InputIteratorTransformer (61) - +- ShuffleQueryStage (59), Statistics(X) - +- ColumnarExchange (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) - :- ^ ProjectExecTransformer (39) - : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) - : :- ^ ProjectExecTransformer (30) - : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) - : : :- ^ ProjectExecTransformer (21) - : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) - : : : :- ^ ProjectExecTransformer (12) - : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- BroadcastQueryStage (6), Statistics(X) - : : : : : +- ColumnarBroadcastExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (19) - : : : +- BroadcastQueryStage (17), Statistics(X) - : : : +- ColumnarBroadcastExchange (16) - : : : +- ^ FilterExecTransformer (14) - : : : +- ^ Scan parquet (13) - : : +- ^ InputIteratorTransformer (28) - : : +- BroadcastQueryStage (26), Statistics(X) - : : +- ColumnarBroadcastExchange (25) - : : +- ^ FilterExecTransformer (23) - : : +- ^ Scan parquet (22) - : +- ^ InputIteratorTransformer (37) - : +- BroadcastQueryStage (35), Statistics(X) - : +- ColumnarBroadcastExchange (34) - : +- ^ FilterExecTransformer (32) - : +- ^ Scan parquet (31) - +- ^ InputIteratorTransformer (46) - +- BroadcastQueryStage (44), Statistics(X) - +- ColumnarBroadcastExchange (43) - +- ^ FilterExecTransformer (41) - +- ^ Scan parquet (40) + VeloxColumnarToRowExec (65) + +- ^ SortExecTransformer (63) + +- ^ InputIteratorTransformer (62) + +- ShuffleQueryStage (60), Statistics(X) + +- ColumnarExchange (59) + +- VeloxAppendBatches (58) + +- ^ RegularHashAggregateExecTransformer (56) + +- ^ InputIteratorTransformer (55) + +- ShuffleQueryStage (53), Statistics(X) + +- ColumnarExchange (52) + +- ^ ProjectExecTransformer (50) + +- ^ FlushableHashAggregateExecTransformer (49) + +- ^ ProjectExecTransformer (48) + +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (47) + :- ^ ProjectExecTransformer (39) + : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (38) + : :- ^ ProjectExecTransformer (30) + : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (29) + : : :- ^ ProjectExecTransformer (21) + : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildRight (20) + : : : :- ^ ProjectExecTransformer (12) + : : : : +- ^ BroadcastHashJoinExecTransformer Inner BuildLeft (11) + : : : : :- ^ InputIteratorTransformer (8) + : : : : : +- BroadcastQueryStage (6), Statistics(X) + : : : : : +- ColumnarBroadcastExchange (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ FilterExecTransformer (10) + : : : : +- ^ Scan parquet (9) + : : : +- ^ InputIteratorTransformer (19) + : : : +- BroadcastQueryStage (17), Statistics(X) + : : : +- ColumnarBroadcastExchange (16) + : : : +- ^ FilterExecTransformer (14) + : : : +- ^ Scan parquet (13) + : : +- ^ InputIteratorTransformer (28) + : : +- BroadcastQueryStage (26), Statistics(X) + : : +- ColumnarBroadcastExchange (25) + : : +- ^ FilterExecTransformer (23) + : : +- ^ Scan parquet (22) + : +- ^ InputIteratorTransformer (37) + : +- BroadcastQueryStage (35), Statistics(X) + : +- ColumnarBroadcastExchange (34) + : +- ^ FilterExecTransformer (32) + : +- ^ Scan parquet (31) + +- ^ InputIteratorTransformer (46) + +- BroadcastQueryStage (44), Statistics(X) + +- ColumnarBroadcastExchange (43) + +- ^ FilterExecTransformer (41) + +- ^ Scan parquet (40) +- == Initial Plan == - Sort (97) - +- Exchange (96) - +- HashAggregate (95) - +- Exchange (94) - +- HashAggregate (93) - +- Project (92) - +- BroadcastHashJoin Inner BuildRight (91) - :- Project (87) - : +- BroadcastHashJoin Inner BuildRight (86) - : :- Project (82) - : : +- BroadcastHashJoin Inner BuildRight (81) - : : :- Project (77) - : : : +- BroadcastHashJoin Inner BuildRight (76) - : : : :- Project (72) - : : : : +- BroadcastHashJoin Inner BuildLeft (71) - : : : : :- BroadcastExchange (68) - : : : : : +- Project (67) - : : : : : +- Filter (66) - : : : : : +- Scan parquet (65) - : : : : +- Filter (70) - : : : : +- Scan parquet (69) - : : : +- BroadcastExchange (75) - : : : +- Filter (74) - : : : +- Scan parquet (73) - : : +- BroadcastExchange (80) - : : +- Filter (79) - : : +- Scan parquet (78) - : +- BroadcastExchange (85) - : +- Filter (84) - : +- Scan parquet (83) - +- BroadcastExchange (90) - +- Filter (89) - +- Scan parquet (88) + Sort (98) + +- Exchange (97) + +- HashAggregate (96) + +- Exchange (95) + +- HashAggregate (94) + +- Project (93) + +- BroadcastHashJoin Inner BuildRight (92) + :- Project (88) + : +- BroadcastHashJoin Inner BuildRight (87) + : :- Project (83) + : : +- BroadcastHashJoin Inner BuildRight (82) + : : :- Project (78) + : : : +- BroadcastHashJoin Inner BuildRight (77) + : : : :- Project (73) + : : : : +- BroadcastHashJoin Inner BuildLeft (72) + : : : : :- BroadcastExchange (69) + : : : : : +- Project (68) + : : : : : +- Filter (67) + : : : : : +- Scan parquet (66) + : : : : +- Filter (71) + : : : : +- Scan parquet (70) + : : : +- BroadcastExchange (76) + : : : +- Filter (75) + : : : +- Scan parquet (74) + : : +- BroadcastExchange (81) + : : +- Filter (80) + : : +- Scan parquet (79) + : +- BroadcastExchange (86) + : +- Filter (85) + : +- Scan parquet (84) + +- BroadcastExchange (91) + +- Filter (90) + +- Scan parquet (89) (1) Scan parquet @@ -336,197 +337,201 @@ Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(58) ColumnarExchange +(58) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(59) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(60) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(60) InputAdapter +(61) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(61) InputIteratorTransformer +(62) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(62) SortExecTransformer +(63) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(63) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(64) VeloxColumnarToRowExec +(65) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(65) Scan parquet +(66) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(66) Filter +(67) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(67) Project +(68) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(68) BroadcastExchange +(69) BroadcastExchange Input [1]: [p_partkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),false), [plan_id=X] -(69) Scan parquet +(70) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(70) Filter +(71) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(71) BroadcastHashJoin +(72) BroadcastHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(72) Project +(73) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(73) Scan parquet +(74) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(74) Filter +(75) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(75) BroadcastExchange +(76) BroadcastExchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(76) BroadcastHashJoin +(77) BroadcastHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(77) Project +(78) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(78) Scan parquet +(79) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(79) Filter +(80) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(80) BroadcastExchange +(81) BroadcastExchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, false], input[0, bigint, false]),false), [plan_id=X] -(81) BroadcastHashJoin +(82) BroadcastHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(82) Project +(83) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(83) Scan parquet +(84) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(84) Filter +(85) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(85) BroadcastExchange +(86) BroadcastExchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(86) BroadcastHashJoin +(87) BroadcastHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(87) Project +(88) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(88) Scan parquet +(89) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(89) Filter +(90) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(90) BroadcastExchange +(91) BroadcastExchange Input [2]: [n_nationkey#X, n_name#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, false]),false), [plan_id=X] -(91) BroadcastHashJoin +(92) BroadcastHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(92) Project +(93) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(93) HashAggregate +(94) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(94) Exchange +(95) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(95) HashAggregate +(96) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(96) Exchange +(97) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) Sort +(98) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(98) AdaptiveSparkPlan +(99) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt index 9316f6f8ebb50..69a930ea8545d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt index a09e6167ffd3c..5d0c6e67cb0d5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/10.txt @@ -1,78 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- TakeOrderedAndProjectExecTransformer (65) + +- ^ ProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ShuffleQueryStage (59) + +- ColumnarExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ NoopFilter (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -94,358 +100,382 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(57) ProjectExecTransformer +(63) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(65) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt index 47b241e9a343f..eecba8c53af69 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/11.txt @@ -1,65 +1,70 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (77) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ FilterExecTransformer (46) + +- ^ RegularHashAggregateExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42) + +- ColumnarExchange (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (76) + +- Exchange (75) + +- Filter (74) + +- HashAggregate (73) + +- Exchange (72) + +- HashAggregate (71) + +- Project (70) + +- ShuffledHashJoin Inner BuildRight (69) + :- Exchange (64) + : +- Project (63) + : +- ShuffledHashJoin Inner BuildRight (62) + : :- Exchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- Exchange (61) + : +- Filter (60) + : +- Scan parquet (59) + +- Exchange (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) (1) Scan parquet @@ -81,292 +86,312 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(43) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(42) FilterExecTransformer +(46) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(48) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(49) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(51) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(53) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(56) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(58) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(59) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(61) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(62) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(63) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(64) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(67) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(68) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(70) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(71) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(72) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(73) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(69) Filter +(74) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(75) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(76) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(77) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt index 9adf1b6060ef7..e6f809047498d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/12.txt @@ -1,46 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- ShuffledHashJoin Inner BuildLeft (45) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -62,196 +65,208 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(38) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(39) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(40) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(43) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(44) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(46) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(48) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(50) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(51) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt index fb354643ede9e..b716b8f475ba9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftOuter BuildRight (46) + :- Exchange (41) + : +- Scan parquet (40) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -60,223 +63,235 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(45) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(47) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt index 62a2d4a7b6174..00b5fb4142f38 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,144 +54,152 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X, p_type#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt index 791ededdabda2..dbb20979b70ef 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/15.txt @@ -1,44 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (47) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- ShuffledHashJoin Inner BuildLeft (41) - :- Exchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (46) + +- Exchange (45) + +- Project (44) + +- ShuffledHashJoin Inner BuildLeft (43) + :- Exchange (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Filter (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- Filter (37) + +- Scan parquet (36) (1) Scan parquet @@ -60,182 +62,190 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(20) FilterExecTransformer +(21) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(26) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) SortExecTransformer +(30) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(30) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(31) Scan parquet +(33) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(33) Exchange +(35) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Scan parquet +(36) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(37) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(38) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(40) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(40) Filter +(42) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(42) Project +(44) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(45) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(46) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(47) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt index 2060266a25507..e3570601883a4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/16.txt @@ -1,59 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (45) + +- ^ SortExecTransformer (43) + +- ^ InputIteratorTransformer (42) + +- ShuffleQueryStage (40) + +- ColumnarExchange (39) + +- VeloxAppendBatches (38) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- ^ ProjectExecTransformer (30) + +- ^ FlushableHashAggregateExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (66) + +- Exchange (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- ShuffledHashJoin Inner BuildRight (57) + :- Exchange (53) + : +- BroadcastHashJoin LeftAnti BuildRight (52) + : :- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -75,270 +78,282 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(29) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(38) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(39) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(41) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(43) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(48) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(49) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(50) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(51) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(53) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(56) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(58) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(59) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(62) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(63) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(66) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt index 504ee433e9e8a..b79bd948bec82 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/17.txt @@ -1,56 +1,58 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ NoopFilter (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ ProjectExecTransformer (34) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ NoopFilter (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Project (48) + : +- ShuffledHashJoin Inner BuildRight (47) + : :- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -72,250 +74,258 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) NoopFilter +(22) NoopFilter Input [2]: [l_partkey#X, l_quantity#X] Arguments: [l_partkey#X, l_quantity#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(32) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(37) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(40) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(41) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(42) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(43) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(45) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(46) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(56) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(58) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt index 78efb8c674702..a7a179a815548 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/18.txt @@ -1,89 +1,94 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51) - +- ReusedExchange (50) + VeloxColumnarToRowExec (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ RegularHashAggregateExecTransformer (65) + +- ^ ProjectExecTransformer (64) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) + :- ^ InputIteratorTransformer (45) + : +- ShuffleQueryStage (43) + : +- ColumnarExchange (42) + : +- VeloxAppendBatches (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (37) + : +- ShuffleQueryStage (35) + : +- ColumnarExchange (34) + : +- VeloxAppendBatches (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ RegularHashAggregateExecTransformer (27) + : +- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + :- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- VeloxAppendBatches (50) + : +- ^ ProjectExecTransformer (48) + : +- ^ NoopFilter (47) + : +- ^ Scan parquet (46) + +- ^ ProjectExecTransformer (61) + +- ^ FilterExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56) + +- ReusedExchange (55) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (101) + +- HashAggregate (100) + +- HashAggregate (99) + +- Project (98) + +- ShuffledHashJoin Inner BuildRight (97) + :- Exchange (86) + : +- Project (85) + : +- ShuffledHashJoin Inner BuildLeft (84) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (83) + : +- ShuffledHashJoin LeftSemi BuildRight (82) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Project (81) + : +- Filter (80) + : +- HashAggregate (79) + : +- Exchange (78) + : +- HashAggregate (77) + : +- Scan parquet (76) + +- ShuffledHashJoin LeftSemi BuildRight (96) + :- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Project (95) + +- Filter (94) + +- HashAggregate (93) + +- Exchange (92) + +- HashAggregate (91) + +- Scan parquet (90) (1) Scan parquet @@ -105,420 +110,440 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(36) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(38) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(41) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(42) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(43) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(44) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(45) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) NoopFilter +(47) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(50) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(51) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(52) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(53) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(54) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(55) ReusedExchange [Reuses operator id: 23] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(57) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(60) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(61) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(64) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(65) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(68) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(70) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(72) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(73) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(76) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(77) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(78) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(79) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(80) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(81) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(83) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(84) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(85) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(86) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(91) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(92) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(93) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(95) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(98) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(99) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(100) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(101) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt index aa13d1509c29f..6ec9ae965ee99 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,140 +53,148 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt index 959bcd4ef7031..24be4842e1b8b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/20.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (126) +AdaptiveSparkPlan (136) +- == Final Plan == - VeloxColumnarToRowExec (86) - +- ^ SortExecTransformer (84) - +- ^ InputIteratorTransformer (83) - +- ShuffleQueryStage (81) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ NoopFilter (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ NoopFilter (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ NoopFilter (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (96) + +- ^ SortExecTransformer (94) + +- ^ InputIteratorTransformer (93) + +- ShuffleQueryStage (91) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ NoopFilter (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ NoopFilter (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (125) - +- Exchange (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin LeftSemi BuildRight (115) - : :- Exchange (89) - : : +- Filter (88) - : : +- Scan parquet (87) - : +- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin Inner BuildLeft (112) - : :- Exchange (98) - : : +- ShuffledHashJoin LeftSemi BuildRight (97) - : : :- Exchange (92) - : : : +- Filter (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (111) - : +- Filter (110) - : +- HashAggregate (109) - : +- HashAggregate (108) - : +- ShuffledHashJoin LeftSemi BuildRight (107) - : :- Exchange (102) - : : +- Project (101) - : : +- Filter (100) - : : +- Scan parquet (99) - : +- Exchange (106) - : +- Project (105) - : +- Filter (104) - : +- Scan parquet (103) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (135) + +- Exchange (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin LeftSemi BuildRight (125) + : :- Exchange (99) + : : +- Filter (98) + : : +- Scan parquet (97) + : +- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin Inner BuildLeft (122) + : :- Exchange (108) + : : +- ShuffledHashJoin LeftSemi BuildRight (107) + : : :- Exchange (102) + : : : +- Filter (101) + : : : +- Scan parquet (100) + : : +- Exchange (106) + : : +- Project (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (121) + : +- Filter (120) + : +- HashAggregate (119) + : +- HashAggregate (118) + : +- ShuffledHashJoin LeftSemi BuildRight (117) + : :- Exchange (112) + : : +- Project (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- Exchange (116) + : +- Project (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (131) + +- Project (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,518 +136,558 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) NoopFilter +(20) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) NoopFilter +(37) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(78) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) InputAdapter +(92) InputAdapter Input [2]: [s_name#X, s_address#X] -(83) InputIteratorTransformer +(93) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(84) SortExecTransformer +(94) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(85) WholeStageCodegenTransformer (X) +(95) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(86) VeloxColumnarToRowExec +(96) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(87) Scan parquet +(97) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(89) Exchange +(99) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(91) Filter +(101) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(92) Exchange +(102) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(103) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(94) Filter +(104) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(95) Project +(105) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(96) Exchange +(106) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(98) Exchange +(108) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(109) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(110) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(101) Project +(111) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(102) Exchange +(112) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(113) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(104) Filter +(114) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(105) Project +(115) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(106) Exchange +(116) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(108) HashAggregate +(118) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(109) HashAggregate +(119) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(110) Filter +(120) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(111) Exchange +(121) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(113) Project +(123) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(114) Exchange +(124) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(116) Project +(126) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(117) Exchange +(127) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(120) Project +(130) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(131) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(123) Project +(133) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(124) Exchange +(134) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(135) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(126) AdaptiveSparkPlan +(136) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt index e9418fc2a71cd..0099ce88edc8c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/21.txt @@ -1,104 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (119) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- TakeOrderedAndProjectExecTransformer (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (91) + +- TakeOrderedAndProjectExecTransformer (90) + +- ^ RegularHashAggregateExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85) + +- ColumnarExchange (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ NoopFilter (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ NoopFilter (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ NoopFilter (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (118) - +- HashAggregate (117) - +- Exchange (116) - +- HashAggregate (115) - +- Project (114) - +- ShuffledHashJoin Inner BuildRight (113) - :- Exchange (108) - : +- Project (107) - : +- ShuffledHashJoin Inner BuildRight (106) - : :- Exchange (101) - : : +- Project (100) - : : +- ShuffledHashJoin Inner BuildLeft (99) - : : :- Exchange (85) - : : : +- Filter (84) - : : : +- Scan parquet (83) - : : +- Exchange (98) - : : +- ShuffledHashJoin LeftAnti BuildRight (97) - : : :- ShuffledHashJoin LeftSemi BuildRight (92) - : : : :- Exchange (89) - : : : : +- Project (88) - : : : : +- Filter (87) - : : : : +- Scan parquet (86) - : : : +- Exchange (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (105) - : +- Project (104) - : +- Filter (103) - : +- Scan parquet (102) - +- Exchange (112) - +- Project (111) - +- Filter (110) - +- Scan parquet (109) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildLeft (108) + : : :- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Exchange (107) + : : +- ShuffledHashJoin LeftAnti BuildRight (106) + : : :- ShuffledHashJoin LeftSemi BuildRight (101) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Exchange (100) + : : : +- Scan parquet (99) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Exchange (114) + : +- Project (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -120,494 +129,530 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) NoopFilter +(29) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) NoopFilter +(54) NoopFilter Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: [o_orderkey#X, o_orderstatus#X] -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) NoopFilter +(71) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(85) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(86) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(87) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(88) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) TakeOrderedAndProjectExecTransformer +(90) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(82) VeloxColumnarToRowExec +(91) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(83) Scan parquet +(92) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(84) Filter +(93) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(85) Exchange +(94) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(87) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(88) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(89) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(99) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(93) Scan parquet +(102) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(94) Filter +(103) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(95) Project +(104) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(96) Exchange +(105) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(98) Exchange +(107) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(100) Project +(109) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(101) Exchange +(110) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(103) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(104) Project +(113) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(105) Exchange +(114) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(107) Project +(116) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(108) Exchange +(117) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(110) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(111) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(112) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(114) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(115) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(116) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(118) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(119) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt index 2b0fcd16aadc7..ac5dfa762fa1b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/22.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftAnti BuildRight (42) + :- Exchange (39) + : +- Filter (38) + : +- Scan parquet (37) + +- Exchange (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,182 +62,194 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(29) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(30) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(31) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(32) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(33) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(34) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(37) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(38) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(39) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(40) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(43) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(44) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(45) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(46) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(47) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(48) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt index ed7b3ea7d3771..f188fa96b0d81 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,244 +76,260 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt index 07a00316284a8..0a7ae2856c03c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/4.txt @@ -1,47 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftSemi BuildRight (46) + :- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -63,200 +66,212 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(40) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(44) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(45) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(47) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(48) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(49) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(51) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(52) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt index 766f9f5a13147..8b26063b15279 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/5.txt @@ -1,115 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- Filter (110) + : : : : +- Scan parquet (109) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -131,552 +142,596 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(99) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(109) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(110) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(111) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(114) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(115) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(118) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(120) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(121) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(124) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(126) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(127) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(130) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(132) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(133) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(136) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(137) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(139) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(140) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(141) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(142) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(132) Exchange +(143) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(144) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt index 7aae3dccfd9ba..a5022343944f9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt index 67c25166ae14a..a7a157760c7c6 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/7.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70) - +- ReusedExchange (69) + VeloxColumnarToRowExec (100) + +- ^ SortExecTransformer (98) + +- ^ InputIteratorTransformer (97) + +- ShuffleQueryStage (95) + +- ColumnarExchange (94) + +- VeloxAppendBatches (93) + +- ^ RegularHashAggregateExecTransformer (91) + +- ^ InputIteratorTransformer (90) + +- ShuffleQueryStage (88) + +- ColumnarExchange (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- ShuffledHashJoin Inner BuildRight (131) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin Inner BuildRight (125) + : :- Exchange (121) + : : +- Project (120) + : : +- ShuffledHashJoin Inner BuildRight (119) + : : :- Exchange (115) + : : : +- Project (114) + : : : +- ShuffledHashJoin Inner BuildRight (113) + : : : :- Exchange (109) + : : : : +- Project (108) + : : : : +- ShuffledHashJoin Inner BuildLeft (107) + : : : : :- Exchange (103) + : : : : : +- Filter (102) + : : : : : +- Scan parquet (101) + : : : : +- Exchange (106) + : : : : +- Filter (105) + : : : : +- Scan parquet (104) + : : : +- Exchange (112) + : : : +- Filter (111) + : : : +- Scan parquet (110) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Exchange (124) + : +- Filter (123) + : +- Scan parquet (122) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,524 +136,564 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X, o_custkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(88) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(89) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(90) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(91) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(92) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(93) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(94) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(95) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(96) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(97) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(98) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(99) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(100) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(101) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(102) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(103) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(104) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(105) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(106) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(108) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(114) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(115) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(116) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(117) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(118) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(120) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(121) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(122) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(123) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(124) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(126) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(127) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(130) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt index 4c1fb04b86609..240315c7973c1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/8.txt @@ -1,150 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (192) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ NoopFilter (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ NoopFilter (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ NoopFilter (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ NoopFilter (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ NoopFilter (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ NoopFilter (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (140) + +- ^ SortExecTransformer (138) + +- ^ InputIteratorTransformer (137) + +- ShuffleQueryStage (135) + +- ColumnarExchange (134) + +- VeloxAppendBatches (133) + +- ^ ProjectExecTransformer (131) + +- ^ RegularHashAggregateExecTransformer (130) + +- ^ InputIteratorTransformer (129) + +- ShuffleQueryStage (127) + +- ColumnarExchange (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ NoopFilter (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ NoopFilter (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ NoopFilter (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ NoopFilter (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ NoopFilter (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ NoopFilter (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (191) + +- Exchange (190) + +- HashAggregate (189) + +- Exchange (188) + +- HashAggregate (187) + +- Project (186) + +- ShuffledHashJoin Inner BuildRight (185) + :- Exchange (180) + : +- Project (179) + : +- ShuffledHashJoin Inner BuildRight (178) + : :- Exchange (174) + : : +- Project (173) + : : +- ShuffledHashJoin Inner BuildRight (172) + : : :- Exchange (168) + : : : +- Project (167) + : : : +- ShuffledHashJoin Inner BuildRight (166) + : : : :- Exchange (162) + : : : : +- Project (161) + : : : : +- ShuffledHashJoin Inner BuildRight (160) + : : : : :- Exchange (156) + : : : : : +- Project (155) + : : : : : +- ShuffledHashJoin Inner BuildRight (154) + : : : : : :- Exchange (150) + : : : : : : +- Project (149) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) + : : : : : : :- Exchange (144) + : : : : : : : +- Project (143) + : : : : : : : +- Filter (142) + : : : : : : : +- Scan parquet (141) + : : : : : : +- Exchange (147) + : : : : : : +- Filter (146) + : : : : : : +- Scan parquet (145) + : : : : : +- Exchange (153) + : : : : : +- Filter (152) + : : : : : +- Scan parquet (151) + : : : : +- Exchange (159) + : : : : +- Filter (158) + : : : : +- Scan parquet (157) + : : : +- Exchange (165) + : : : +- Filter (164) + : : : +- Scan parquet (163) + : : +- Exchange (171) + : : +- Filter (170) + : : +- Scan parquet (169) + : +- Exchange (177) + : +- Filter (176) + : +- Scan parquet (175) + +- Exchange (184) + +- Project (183) + +- Filter (182) + +- Scan parquet (181) (1) Scan parquet @@ -166,732 +181,792 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X, n_regionkey#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) NoopFilter +(96) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) NoopFilter +(113) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(127) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(128) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(129) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(130) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(131) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(132) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(133) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(134) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(135) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(136) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(137) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(138) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(139) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(140) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(141) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(142) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(143) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(144) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(145) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(146) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(147) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(148) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(149) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(150) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(151) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(152) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(153) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(154) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(155) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(156) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(157) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(158) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(159) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(160) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(161) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(162) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(163) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(164) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(165) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(166) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(167) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(168) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(169) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(170) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(171) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(172) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(173) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(174) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(175) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(176) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(177) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(178) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(179) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(180) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(181) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(182) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(183) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(184) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(185) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(186) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(187) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(188) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(189) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(175) Exchange +(190) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(191) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(192) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt index 582d323dccb27..2bbb889c2ae3f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark32/9.txt @@ -1,114 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- ShuffledHashJoin Inner BuildRight (137) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (109) + : : : : : +- Project (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -130,548 +141,592 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_orderdate#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(99) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(108) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(109) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(110) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(111) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(112) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(114) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(115) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(120) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(121) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(124) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(126) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(127) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(130) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(132) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(133) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(136) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(138) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(139) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(140) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(141) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(142) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(143) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt index 63b30bb5d26b0..5ad7bc9d3675d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt index 6213f61ffdf42..f83e7572840a7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/10.txt @@ -1,78 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- TakeOrderedAndProjectExecTransformer (65) + +- ^ ProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ NoopFilter (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -94,358 +100,382 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(57) ProjectExecTransformer +(63) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(65) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt index 80dbdb2c50ad1..b3fb729d8d967 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/11.txt @@ -1,65 +1,70 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (77) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ FilterExecTransformer (46) + +- ^ RegularHashAggregateExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (76) + +- Exchange (75) + +- Filter (74) + +- HashAggregate (73) + +- Exchange (72) + +- HashAggregate (71) + +- Project (70) + +- ShuffledHashJoin Inner BuildRight (69) + :- Exchange (64) + : +- Project (63) + : +- ShuffledHashJoin Inner BuildRight (62) + : :- Exchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- Exchange (61) + : +- Filter (60) + : +- Scan parquet (59) + +- Exchange (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) (1) Scan parquet @@ -81,549 +86,579 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(43) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(42) FilterExecTransformer +(46) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(48) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(49) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(51) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(53) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(56) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(58) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(59) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(61) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(62) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(63) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(64) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(67) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(68) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(70) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(71) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(72) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(73) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(69) Filter +(74) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(75) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(76) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(77) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - :- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ NoopFilter (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (109) + +- ^ ProjectExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ RegularHashAggregateExecTransformer (105) + +- ^ ProjectExecTransformer (104) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (103) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- VeloxAppendBatches (94) + : +- ^ ProjectExecTransformer (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (91) + : :- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (90) + : +- ShuffleQueryStage (88), Statistics(X) + : +- ReusedExchange (87) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ReusedExchange (99) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (126) + +- HashAggregate (125) + +- Project (124) + +- ShuffledHashJoin Inner BuildRight (123) + :- Exchange (118) + : +- Project (117) + : +- ShuffledHashJoin Inner BuildRight (116) + : :- Exchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Exchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) + + +(78) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) NoopFilter +(79) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(75) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(82) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(83) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(84) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(85) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(86) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(87) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(88) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(89) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(90) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(91) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(86) ProjectExecTransformer +(92) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(99) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(100) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(101) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(102) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(103) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(97) ProjectExecTransformer +(104) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(105) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(100) ProjectExecTransformer +(107) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(101) WholeStageCodegenTransformer (X) +(108) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(109) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(110) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(111) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(112) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(113) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(114) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(115) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(110) Project +(117) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(118) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(123) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(124) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(125) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(126) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt index 58a8788ca2527..c9a5452d9aa0e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/12.txt @@ -1,46 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- ShuffledHashJoin Inner BuildLeft (45) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -62,196 +65,208 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(38) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(39) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(40) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(43) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(44) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(46) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(48) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(50) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(51) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt index 8837c6ef41432..0a402001c23bd 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftOuter BuildRight (46) + :- Exchange (41) + : +- Scan parquet (40) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -60,223 +63,235 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(45) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(47) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt index 2c06906d4fb87..cb3ddbb3a2f7b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,144 +54,152 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X, p_type#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt index 270cea0f6f46e..7f0635c0d9a42 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (29) + +- AQEShuffleRead (28) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- Project (41) + +- ShuffledHashJoin Inner BuildLeft (40) + :- Exchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Filter (39) + +- HashAggregate (38) + +- Exchange (37) + +- HashAggregate (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -59,328 +61,336 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(20) FilterExecTransformer +(21) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(26) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(28) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(30) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(32) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(33) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(35) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(37) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(37) Filter +(39) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(40) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(39) Project +(41) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(42) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ NoopFilter (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (48) + +- ^ ProjectExecTransformer (47) + +- ^ NoopFilter (46) + +- ^ Scan parquet (45) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(43) Scan parquet +(45) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) NoopFilter +(46) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(48) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt index 90c77f0de9b30..b9a3fb8dadd08 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/16.txt @@ -1,59 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (45) + +- ^ SortExecTransformer (43) + +- ^ InputIteratorTransformer (42) + +- ShuffleQueryStage (40), Statistics(X) + +- ColumnarExchange (39) + +- VeloxAppendBatches (38) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ ProjectExecTransformer (30) + +- ^ FlushableHashAggregateExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (66) + +- Exchange (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- ShuffledHashJoin Inner BuildRight (57) + :- Exchange (53) + : +- BroadcastHashJoin LeftAnti BuildRight (52) + : :- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -75,270 +78,282 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(29) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(38) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(39) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(41) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(43) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(48) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(49) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(50) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(51) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(53) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(56) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(58) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(59) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(62) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(63) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(66) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt index 16321ffa4e59f..6206907a85886 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/17.txt @@ -1,56 +1,58 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ NoopFilter (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ ProjectExecTransformer (34) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ NoopFilter (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Project (48) + : +- ShuffledHashJoin Inner BuildRight (47) + : :- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -72,250 +74,258 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) NoopFilter +(22) NoopFilter Input [2]: [l_partkey#X, l_quantity#X] Arguments: [l_partkey#X, l_quantity#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(32) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(37) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(40) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(41) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(42) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(43) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(45) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(46) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(56) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(58) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt index fa441bac82ccf..038422e0fecfb 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/18.txt @@ -1,89 +1,94 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ RegularHashAggregateExecTransformer (65) + +- ^ ProjectExecTransformer (64) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) + :- ^ InputIteratorTransformer (45) + : +- ShuffleQueryStage (43), Statistics(X) + : +- ColumnarExchange (42) + : +- VeloxAppendBatches (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (37) + : +- ShuffleQueryStage (35), Statistics(X) + : +- ColumnarExchange (34) + : +- VeloxAppendBatches (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ RegularHashAggregateExecTransformer (27) + : +- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + :- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- VeloxAppendBatches (50) + : +- ^ ProjectExecTransformer (48) + : +- ^ NoopFilter (47) + : +- ^ Scan parquet (46) + +- ^ ProjectExecTransformer (61) + +- ^ FilterExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ReusedExchange (55) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (101) + +- HashAggregate (100) + +- HashAggregate (99) + +- Project (98) + +- ShuffledHashJoin Inner BuildRight (97) + :- Exchange (86) + : +- Project (85) + : +- ShuffledHashJoin Inner BuildLeft (84) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (83) + : +- ShuffledHashJoin LeftSemi BuildRight (82) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Project (81) + : +- Filter (80) + : +- HashAggregate (79) + : +- Exchange (78) + : +- HashAggregate (77) + : +- Scan parquet (76) + +- ShuffledHashJoin LeftSemi BuildRight (96) + :- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Project (95) + +- Filter (94) + +- HashAggregate (93) + +- Exchange (92) + +- HashAggregate (91) + +- Scan parquet (90) (1) Scan parquet @@ -105,420 +110,440 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(36) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(38) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(41) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(42) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(43) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(44) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(45) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) NoopFilter +(47) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(50) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(51) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(52) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(53) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(54) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(55) ReusedExchange [Reuses operator id: 23] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(57) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(60) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(61) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(64) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(65) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(68) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(70) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(72) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(73) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(76) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(77) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(78) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(79) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(80) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(81) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(83) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(84) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(85) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(86) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(91) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(92) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(93) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(95) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(98) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(99) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(100) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(101) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt index d9ceedf0db6c3..440383aa1cd72 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,140 +53,148 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt index 305f98339a440..d3cff30e0ce28 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/20.txt @@ -1,109 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ NoopFilter (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ NoopFilter (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ NoopFilter (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (93) + +- AQEShuffleRead (92) + +- ShuffleQueryStage (91), Statistics(X) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74), Statistics(X) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58), Statistics(X) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ NoopFilter (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ NoopFilter (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (132) + +- Exchange (131) + +- Project (130) + +- ShuffledHashJoin Inner BuildRight (129) + :- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin LeftSemi BuildRight (122) + : :- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (121) + : +- Project (120) + : +- ShuffledHashJoin Inner BuildLeft (119) + : :- Exchange (105) + : : +- ShuffledHashJoin LeftSemi BuildRight (104) + : : :- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Exchange (118) + : +- Filter (117) + : +- HashAggregate (116) + : +- HashAggregate (115) + : +- ShuffledHashJoin LeftSemi BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (128) + +- Project (127) + +- Filter (126) + +- Scan parquet (125) (1) Scan parquet @@ -125,508 +135,548 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) NoopFilter +(20) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) NoopFilter +(37) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(78) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(92) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(93) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(94) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(95) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(96) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(97) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(99) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(95) Exchange +(105) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(106) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(107) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(108) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(112) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(113) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(105) HashAggregate +(115) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(116) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(117) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(118) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(113) Project +(123) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(124) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(125) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(126) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(127) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(128) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(129) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(120) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt index 04cc1f47a3d3e..d2c18146af4fe 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/21.txt @@ -1,103 +1,112 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (90) + +- ^ RegularHashAggregateExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ NoopFilter (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ NoopFilter (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ NoopFilter (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- ShuffledHashJoin Inner BuildLeft (107) + : : :- Exchange (93) + : : : +- Filter (92) + : : : +- Scan parquet (91) + : : +- Exchange (106) + : : +- ShuffledHashJoin LeftAnti BuildRight (105) + : : :- ShuffledHashJoin LeftSemi BuildRight (100) + : : : :- Exchange (97) + : : : : +- Project (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (99) + : : : +- Scan parquet (98) + : : +- Exchange (104) + : : +- Project (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -119,490 +128,526 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) NoopFilter +(29) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) NoopFilter +(54) NoopFilter Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: [o_orderkey#X, o_orderstatus#X] -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) NoopFilter +(71) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(85) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(86) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(87) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(88) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(91) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(92) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(93) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(94) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(95) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(96) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(98) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(101) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(102) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(103) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(104) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(106) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(99) Project +(108) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(109) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(111) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(112) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(113) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(106) Project +(115) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(116) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(117) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(118) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(119) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(120) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(122) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(123) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(124) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(125) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(126) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt index 9a513e223197c..ea3839718ac11 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/22.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftAnti BuildRight (42) + :- Exchange (39) + : +- Filter (38) + : +- Scan parquet (37) + +- Exchange (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,296 +62,308 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(29) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(30) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(31) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(32) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(33) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(34) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(37) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(38) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(39) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(40) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(43) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(44) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(45) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(46) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(47) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(48) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ NoopFilter (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (61) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- ^ FlushableHashAggregateExecTransformer (53) + +- ^ ProjectExecTransformer (52) + +- ^ NoopFilter (51) + +- ^ Scan parquet (50) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (67) + +- Exchange (66) + +- HashAggregate (65) + +- Project (64) + +- Filter (63) + +- Scan parquet (62) -(47) Scan parquet +(50) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) NoopFilter +(51) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(49) ProjectExecTransformer +(52) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(53) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(55) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(56) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(57) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(58) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(62) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(63) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(64) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(66) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(67) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt index 0b11cfa1f763e..8b1f048c7d6e7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,244 +76,260 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt index 2eb1f1044104e..054409875b255 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/4.txt @@ -1,47 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftSemi BuildRight (46) + :- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -63,200 +66,212 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(40) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(44) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(45) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(47) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(48) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(49) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(51) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(52) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt index 85b46c76a4998..db0ff094f9205 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/5.txt @@ -1,115 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- Filter (110) + : : : : +- Scan parquet (109) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -131,552 +142,596 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(99) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(109) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(110) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(111) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(114) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(115) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(118) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(120) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(121) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(124) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(126) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(127) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(130) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(132) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(133) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(136) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(137) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(139) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(140) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(141) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(142) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(132) Exchange +(143) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(144) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt index e03830992c2e2..30e1b3fe013b5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt index a2e4435d71880..5bccbe75ed31c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/7.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (100) + +- ^ SortExecTransformer (98) + +- ^ InputIteratorTransformer (97) + +- ShuffleQueryStage (95), Statistics(X) + +- ColumnarExchange (94) + +- VeloxAppendBatches (93) + +- ^ RegularHashAggregateExecTransformer (91) + +- ^ InputIteratorTransformer (90) + +- ShuffleQueryStage (88), Statistics(X) + +- ColumnarExchange (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- ShuffledHashJoin Inner BuildRight (131) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin Inner BuildRight (125) + : :- Exchange (121) + : : +- Project (120) + : : +- ShuffledHashJoin Inner BuildRight (119) + : : :- Exchange (115) + : : : +- Project (114) + : : : +- ShuffledHashJoin Inner BuildRight (113) + : : : :- Exchange (109) + : : : : +- Project (108) + : : : : +- ShuffledHashJoin Inner BuildLeft (107) + : : : : :- Exchange (103) + : : : : : +- Filter (102) + : : : : : +- Scan parquet (101) + : : : : +- Exchange (106) + : : : : +- Filter (105) + : : : : +- Scan parquet (104) + : : : +- Exchange (112) + : : : +- Filter (111) + : : : +- Scan parquet (110) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Exchange (124) + : +- Filter (123) + : +- Scan parquet (122) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,524 +136,564 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X, o_custkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(88) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(89) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(90) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(91) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(92) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(93) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(94) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(95) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(96) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(97) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(98) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(99) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(100) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(101) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(102) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(103) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(104) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(105) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(106) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(108) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(114) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(115) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(116) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(117) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(118) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(120) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(121) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(122) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(123) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(124) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(126) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(127) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(130) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt index 47be7c7230dd2..553bb62a242e4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/8.txt @@ -1,150 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (192) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ NoopFilter (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ NoopFilter (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ NoopFilter (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ NoopFilter (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ NoopFilter (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ NoopFilter (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (140) + +- ^ SortExecTransformer (138) + +- ^ InputIteratorTransformer (137) + +- ShuffleQueryStage (135), Statistics(X) + +- ColumnarExchange (134) + +- VeloxAppendBatches (133) + +- ^ ProjectExecTransformer (131) + +- ^ RegularHashAggregateExecTransformer (130) + +- ^ InputIteratorTransformer (129) + +- ShuffleQueryStage (127), Statistics(X) + +- ColumnarExchange (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ NoopFilter (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ NoopFilter (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ NoopFilter (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ NoopFilter (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ NoopFilter (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ NoopFilter (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (191) + +- Exchange (190) + +- HashAggregate (189) + +- Exchange (188) + +- HashAggregate (187) + +- Project (186) + +- ShuffledHashJoin Inner BuildRight (185) + :- Exchange (180) + : +- Project (179) + : +- ShuffledHashJoin Inner BuildRight (178) + : :- Exchange (174) + : : +- Project (173) + : : +- ShuffledHashJoin Inner BuildRight (172) + : : :- Exchange (168) + : : : +- Project (167) + : : : +- ShuffledHashJoin Inner BuildRight (166) + : : : :- Exchange (162) + : : : : +- Project (161) + : : : : +- ShuffledHashJoin Inner BuildRight (160) + : : : : :- Exchange (156) + : : : : : +- Project (155) + : : : : : +- ShuffledHashJoin Inner BuildRight (154) + : : : : : :- Exchange (150) + : : : : : : +- Project (149) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) + : : : : : : :- Exchange (144) + : : : : : : : +- Project (143) + : : : : : : : +- Filter (142) + : : : : : : : +- Scan parquet (141) + : : : : : : +- Exchange (147) + : : : : : : +- Filter (146) + : : : : : : +- Scan parquet (145) + : : : : : +- Exchange (153) + : : : : : +- Filter (152) + : : : : : +- Scan parquet (151) + : : : : +- Exchange (159) + : : : : +- Filter (158) + : : : : +- Scan parquet (157) + : : : +- Exchange (165) + : : : +- Filter (164) + : : : +- Scan parquet (163) + : : +- Exchange (171) + : : +- Filter (170) + : : +- Scan parquet (169) + : +- Exchange (177) + : +- Filter (176) + : +- Scan parquet (175) + +- Exchange (184) + +- Project (183) + +- Filter (182) + +- Scan parquet (181) (1) Scan parquet @@ -166,732 +181,792 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X, n_regionkey#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) NoopFilter +(96) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) NoopFilter +(113) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(127) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(128) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(129) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(130) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(131) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(132) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(133) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(134) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(135) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(136) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(137) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(138) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(139) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(140) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(141) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(142) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(143) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(144) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(145) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(146) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(147) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(148) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(149) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(150) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(151) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(152) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(153) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(154) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(155) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(156) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(157) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(158) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(159) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(160) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(161) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(162) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(163) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(164) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(165) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(166) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(167) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(168) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(169) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(170) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(171) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(172) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(173) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(174) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(175) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(176) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(177) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(178) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(179) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(180) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(181) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(182) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(183) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(184) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(185) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(186) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(187) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(188) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(189) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(175) Exchange +(190) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(191) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(192) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt index 93001ea6c78df..f894a33ab6e78 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark33/9.txt @@ -1,114 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- ShuffledHashJoin Inner BuildRight (137) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (109) + : : : : : +- Project (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -130,548 +141,592 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_orderdate#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(99) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(108) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(109) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(110) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(111) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(112) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(114) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(115) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(120) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(121) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(124) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(126) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(127) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(130) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(132) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(133) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(136) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(138) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(139) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(140) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(141) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(142) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(143) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt index c254ec8c82caf..589019c4cdbf5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt index 43930a2eb5db0..e26d0ed210e21 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/10.txt @@ -1,78 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ NoopFilter (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ NoopFilter (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- TakeOrderedAndProjectExecTransformer (65) + +- ^ ProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ NoopFilter (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ NoopFilter (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -94,364 +100,388 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(57) ProjectExecTransformer +(63) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(65) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(69) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(76) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt index e8e0cb750c846..a360e2ccdf4d4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/11.txt @@ -1,65 +1,70 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (77) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ FilterExecTransformer (46) + +- ^ RegularHashAggregateExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (76) + +- Exchange (75) + +- Filter (74) + +- HashAggregate (73) + +- Exchange (72) + +- HashAggregate (71) + +- Project (70) + +- ShuffledHashJoin Inner BuildRight (69) + :- Exchange (64) + : +- Project (63) + : +- ShuffledHashJoin Inner BuildRight (62) + : :- Exchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- Exchange (61) + : +- Filter (60) + : +- Scan parquet (59) + +- Exchange (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) (1) Scan parquet @@ -81,557 +86,587 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(43) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(42) FilterExecTransformer +(46) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(48) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(49) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(51) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(53) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(56) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(58) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(59) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(61) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(62) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(58) Project +(63) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(64) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(67) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(68) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(65) Project +(70) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(71) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(72) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(73) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(69) Filter +(74) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(75) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(76) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(77) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - :- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ NoopFilter (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (109) + +- ^ ProjectExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ RegularHashAggregateExecTransformer (105) + +- ^ ProjectExecTransformer (104) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (103) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- VeloxAppendBatches (94) + : +- ^ ProjectExecTransformer (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (91) + : :- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (90) + : +- ShuffleQueryStage (88), Statistics(X) + : +- ReusedExchange (87) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ReusedExchange (99) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (126) + +- HashAggregate (125) + +- Project (124) + +- ShuffledHashJoin Inner BuildRight (123) + :- Exchange (118) + : +- Project (117) + : +- ShuffledHashJoin Inner BuildRight (116) + : :- Exchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Exchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) + + +(78) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) NoopFilter +(79) NoopFilter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(75) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(82) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(83) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(84) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(85) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(86) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(87) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(88) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(89) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(90) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(91) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(86) ProjectExecTransformer +(92) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(99) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(100) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(101) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(102) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(103) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(97) ProjectExecTransformer +(104) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(105) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(100) ProjectExecTransformer +(107) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(101) WholeStageCodegenTransformer (X) +(108) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(109) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(110) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(111) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(112) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(113) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(114) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(115) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(110) Project +(117) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(118) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(123) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(124) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(125) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(126) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt index ea49c8c6402c3..2cd2fbbd94c0c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/12.txt @@ -1,46 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- ShuffledHashJoin Inner BuildLeft (45) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -62,198 +65,210 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(38) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(39) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(40) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(43) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(44) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(43) Project +(46) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(48) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(50) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(51) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt index a7a55ece84ee0..7e296f215aa18 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ NoopFilter (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ NoopFilter (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftOuter BuildRight (46) + :- Exchange (41) + : +- Scan parquet (40) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -60,225 +63,237 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) NoopFilter +(10) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: [o_orderkey#X, o_custkey#X, o_comment#X] -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(45) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(44) Project +(47) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt index 412a680dae784..4080a469e711d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,146 +54,154 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [2]: [p_partkey#X, p_type#X] Arguments: [p_partkey#X, p_type#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt index c66afda3aee43..3fa54d1d647b7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (29) + +- AQEShuffleRead (28) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- Project (41) + +- ShuffledHashJoin Inner BuildLeft (40) + :- Exchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Filter (39) + +- HashAggregate (38) + +- Exchange (37) + +- HashAggregate (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -59,330 +61,338 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(20) FilterExecTransformer +(21) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(26) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(28) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(30) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(32) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(33) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(35) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(37) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(37) Filter +(39) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(40) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(39) Project +(41) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(42) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ NoopFilter (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (48) + +- ^ ProjectExecTransformer (47) + +- ^ NoopFilter (46) + +- ^ Scan parquet (45) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(43) Scan parquet +(45) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) NoopFilter +(46) NoopFilter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(45) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(48) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt index 7bb61df21db4c..70fe9d3d51e48 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/16.txt @@ -1,59 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (45) + +- ^ SortExecTransformer (43) + +- ^ InputIteratorTransformer (42) + +- ShuffleQueryStage (40), Statistics(X) + +- ColumnarExchange (39) + +- VeloxAppendBatches (38) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ ProjectExecTransformer (30) + +- ^ FlushableHashAggregateExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (66) + +- Exchange (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- ShuffledHashJoin Inner BuildRight (57) + :- Exchange (53) + : +- BroadcastHashJoin LeftAnti BuildRight (52) + : :- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -75,273 +78,285 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(29) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(38) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(39) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(41) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(43) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(48) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(49) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(50) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(51) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(50) Exchange +(53) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(56) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(55) Project +(58) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(59) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(62) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(63) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(66) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt index c51280b11b0f3..b5360f9407164 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/17.txt @@ -1,56 +1,58 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ NoopFilter (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ ProjectExecTransformer (34) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ NoopFilter (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Project (48) + : +- ShuffledHashJoin Inner BuildRight (47) + : :- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -72,254 +74,262 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) NoopFilter +(22) NoopFilter Input [2]: [l_partkey#X, l_quantity#X] Arguments: [l_partkey#X, l_quantity#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(32) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(37) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(40) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(41) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(42) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(43) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(45) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(46) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(46) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(56) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(58) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt index 0f3bc73eb8252..557e4f5258cc2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/18.txt @@ -1,89 +1,94 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ NoopFilter (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ NoopFilter (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ RegularHashAggregateExecTransformer (65) + +- ^ ProjectExecTransformer (64) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) + :- ^ InputIteratorTransformer (45) + : +- ShuffleQueryStage (43), Statistics(X) + : +- ColumnarExchange (42) + : +- VeloxAppendBatches (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (37) + : +- ShuffleQueryStage (35), Statistics(X) + : +- ColumnarExchange (34) + : +- VeloxAppendBatches (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ NoopFilter (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ RegularHashAggregateExecTransformer (27) + : +- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + :- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- VeloxAppendBatches (50) + : +- ^ ProjectExecTransformer (48) + : +- ^ NoopFilter (47) + : +- ^ Scan parquet (46) + +- ^ ProjectExecTransformer (61) + +- ^ FilterExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ReusedExchange (55) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (101) + +- HashAggregate (100) + +- HashAggregate (99) + +- Project (98) + +- ShuffledHashJoin Inner BuildRight (97) + :- Exchange (86) + : +- Project (85) + : +- ShuffledHashJoin Inner BuildLeft (84) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (83) + : +- ShuffledHashJoin LeftSemi BuildRight (82) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Project (81) + : +- Filter (80) + : +- HashAggregate (79) + : +- Exchange (78) + : +- HashAggregate (77) + : +- Scan parquet (76) + +- ShuffledHashJoin LeftSemi BuildRight (96) + :- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Project (95) + +- Filter (94) + +- HashAggregate (93) + +- Exchange (92) + +- HashAggregate (91) + +- Scan parquet (90) (1) Scan parquet @@ -105,428 +110,448 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(36) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(38) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(36) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(41) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(42) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(43) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(44) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(45) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) NoopFilter +(47) NoopFilter Input [2]: [l_orderkey#X, l_quantity#X] Arguments: [l_orderkey#X, l_quantity#X] -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(50) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(51) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(52) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(53) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(54) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(55) ReusedExchange [Reuses operator id: 23] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(57) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(60) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(61) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(58) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(59) ProjectExecTransformer +(64) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(65) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(68) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(70) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(72) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(73) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(76) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(77) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(78) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(79) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(80) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(81) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(78) Exchange +(83) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(84) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(80) Project +(85) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(86) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(91) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(92) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(93) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(95) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(93) Project +(98) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(99) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(100) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(101) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt index 1d1169055f322..91187ac8d5a79 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,142 +53,150 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt index 71c30b25fb2bc..01ab88ee0b2be 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/20.txt @@ -1,109 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ NoopFilter (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ NoopFilter (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ NoopFilter (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (93) + +- AQEShuffleRead (92) + +- ShuffleQueryStage (91), Statistics(X) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74), Statistics(X) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ NoopFilter (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ NoopFilter (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58), Statistics(X) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ NoopFilter (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ NoopFilter (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (132) + +- Exchange (131) + +- Project (130) + +- ShuffledHashJoin Inner BuildRight (129) + :- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin LeftSemi BuildRight (122) + : :- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (121) + : +- Project (120) + : +- ShuffledHashJoin Inner BuildLeft (119) + : :- Exchange (105) + : : +- ShuffledHashJoin LeftSemi BuildRight (104) + : : :- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Exchange (118) + : +- Filter (117) + : +- HashAggregate (116) + : +- HashAggregate (115) + : +- ShuffledHashJoin LeftSemi BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (128) + +- Project (127) + +- Filter (126) + +- Scan parquet (125) (1) Scan parquet @@ -125,518 +135,558 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) NoopFilter +(20) NoopFilter Input [2]: [p_partkey#X, p_name#X] Arguments: [p_partkey#X, p_name#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) NoopFilter +(37) NoopFilter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(78) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(92) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(93) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(94) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(95) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(96) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(97) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(99) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(95) Exchange +(105) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(106) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(107) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(108) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(112) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(113) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(105) HashAggregate +(115) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(116) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(117) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(118) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(113) Project +(123) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(124) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(125) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(126) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(127) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(128) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(129) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(120) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt index d9d79a967f032..f57dfa3e6f739 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/21.txt @@ -1,103 +1,112 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ NoopFilter (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ NoopFilter (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ NoopFilter (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ NoopFilter (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (90) + +- ^ RegularHashAggregateExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ NoopFilter (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ NoopFilter (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ NoopFilter (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ NoopFilter (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- ShuffledHashJoin Inner BuildLeft (107) + : : :- Exchange (93) + : : : +- Filter (92) + : : : +- Scan parquet (91) + : : +- Exchange (106) + : : +- ShuffledHashJoin LeftAnti BuildRight (105) + : : :- ShuffledHashJoin LeftSemi BuildRight (100) + : : : :- Exchange (97) + : : : : +- Project (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (99) + : : : +- Scan parquet (98) + : : +- Exchange (104) + : : +- Project (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -119,500 +128,536 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) NoopFilter +(29) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) NoopFilter +(54) NoopFilter Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: [o_orderkey#X, o_orderstatus#X] -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) NoopFilter +(71) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(85) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(86) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(87) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(88) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(91) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(92) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(93) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(94) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(95) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(96) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(98) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(101) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(102) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(103) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(104) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(106) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(99) Project +(108) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(109) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(111) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(112) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(113) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(106) Project +(115) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(116) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(117) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(118) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(119) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(120) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(113) Project +(122) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(123) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(124) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(125) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(126) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt index feeda6c9fded4..1e1b477c5c308 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/22.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftAnti BuildRight (42) + :- Exchange (39) + : +- Filter (38) + : +- Scan parquet (37) + +- Exchange (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,298 +62,310 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(29) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(30) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(31) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(32) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(33) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(34) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(37) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(38) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(39) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(40) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(40) Project +(43) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(44) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(45) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(46) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(47) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(48) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ NoopFilter (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (61) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- ^ FlushableHashAggregateExecTransformer (53) + +- ^ ProjectExecTransformer (52) + +- ^ NoopFilter (51) + +- ^ Scan parquet (50) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (67) + +- Exchange (66) + +- HashAggregate (65) + +- Project (64) + +- Filter (63) + +- Scan parquet (62) -(47) Scan parquet +(50) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) NoopFilter +(51) NoopFilter Input [2]: [c_phone#X, c_acctbal#X] Arguments: [c_phone#X, c_acctbal#X] -(49) ProjectExecTransformer +(52) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(53) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(55) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(56) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(57) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(58) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(62) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(63) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(64) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(66) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(67) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt index ee94233a0a5b6..8c671a61c9f77 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ NoopFilter (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ NoopFilter (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ NoopFilter (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ NoopFilter (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ NoopFilter (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ NoopFilter (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,248 +76,264 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt index 141bd8aa73fdf..08e94f8df743f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/4.txt @@ -1,47 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ NoopFilter (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ NoopFilter (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ NoopFilter (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ NoopFilter (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftSemi BuildRight (46) + :- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -63,202 +66,214 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(40) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(44) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(45) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(44) Project +(47) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(48) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(49) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(51) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(52) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt index 95a5f3ee722fd..888a5f38c53d8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/5.txt @@ -1,115 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- Filter (110) + : : : : +- Scan parquet (109) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -131,562 +142,606 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: [n_nationkey#X, n_name#X, n_regionkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(99) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(109) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(110) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(111) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(103) Project +(114) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(115) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(118) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(109) Project +(120) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(121) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(124) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(115) Project +(126) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(127) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(130) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(121) Project +(132) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(133) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(136) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(137) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(128) Project +(139) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(140) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(141) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(142) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(132) Exchange +(143) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(144) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt index 5987a808f5fd8..fbcc75a247d54 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ NoopFilter (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ NoopFilter (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt index 994d1e163a36f..a7597d3dbe87d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/7.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (100) + +- ^ SortExecTransformer (98) + +- ^ InputIteratorTransformer (97) + +- ShuffleQueryStage (95), Statistics(X) + +- ColumnarExchange (94) + +- VeloxAppendBatches (93) + +- ^ RegularHashAggregateExecTransformer (91) + +- ^ InputIteratorTransformer (90) + +- ShuffleQueryStage (88), Statistics(X) + +- ColumnarExchange (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- ShuffledHashJoin Inner BuildRight (131) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin Inner BuildRight (125) + : :- Exchange (121) + : : +- Project (120) + : : +- ShuffledHashJoin Inner BuildRight (119) + : : :- Exchange (115) + : : : +- Project (114) + : : : +- ShuffledHashJoin Inner BuildRight (113) + : : : :- Exchange (109) + : : : : +- Project (108) + : : : : +- ShuffledHashJoin Inner BuildLeft (107) + : : : : :- Exchange (103) + : : : : : +- Filter (102) + : : : : : +- Scan parquet (101) + : : : : +- Exchange (106) + : : : : +- Filter (105) + : : : : +- Scan parquet (104) + : : : +- Exchange (112) + : : : +- Filter (111) + : : : +- Scan parquet (110) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Exchange (124) + : +- Filter (123) + : +- Scan parquet (122) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,534 +136,574 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [o_orderkey#X, o_custkey#X] Arguments: [o_orderkey#X, o_custkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(88) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(89) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(90) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(91) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(92) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(93) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(94) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(95) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(96) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(97) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(98) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(99) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(100) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(101) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(102) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(103) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(104) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(105) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(106) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(98) Project +(108) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(104) Project +(114) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(115) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(116) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(117) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(118) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(110) Project +(120) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(121) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(122) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(123) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(124) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(116) Project +(126) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(127) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(130) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt index b483283286d48..2e5a862f832f1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/8.txt @@ -1,150 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (192) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ NoopFilter (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ NoopFilter (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ NoopFilter (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ NoopFilter (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ NoopFilter (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ NoopFilter (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ NoopFilter (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ NoopFilter (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (140) + +- ^ SortExecTransformer (138) + +- ^ InputIteratorTransformer (137) + +- ShuffleQueryStage (135), Statistics(X) + +- ColumnarExchange (134) + +- VeloxAppendBatches (133) + +- ^ ProjectExecTransformer (131) + +- ^ RegularHashAggregateExecTransformer (130) + +- ^ InputIteratorTransformer (129) + +- ShuffleQueryStage (127), Statistics(X) + +- ColumnarExchange (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ NoopFilter (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ NoopFilter (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ NoopFilter (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ NoopFilter (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ NoopFilter (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ NoopFilter (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ NoopFilter (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ NoopFilter (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (191) + +- Exchange (190) + +- HashAggregate (189) + +- Exchange (188) + +- HashAggregate (187) + +- Project (186) + +- ShuffledHashJoin Inner BuildRight (185) + :- Exchange (180) + : +- Project (179) + : +- ShuffledHashJoin Inner BuildRight (178) + : :- Exchange (174) + : : +- Project (173) + : : +- ShuffledHashJoin Inner BuildRight (172) + : : :- Exchange (168) + : : : +- Project (167) + : : : +- ShuffledHashJoin Inner BuildRight (166) + : : : :- Exchange (162) + : : : : +- Project (161) + : : : : +- ShuffledHashJoin Inner BuildRight (160) + : : : : :- Exchange (156) + : : : : : +- Project (155) + : : : : : +- ShuffledHashJoin Inner BuildRight (154) + : : : : : :- Exchange (150) + : : : : : : +- Project (149) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) + : : : : : : :- Exchange (144) + : : : : : : : +- Project (143) + : : : : : : : +- Filter (142) + : : : : : : : +- Scan parquet (141) + : : : : : : +- Exchange (147) + : : : : : : +- Filter (146) + : : : : : : +- Scan parquet (145) + : : : : : +- Exchange (153) + : : : : : +- Filter (152) + : : : : : +- Scan parquet (151) + : : : : +- Exchange (159) + : : : : +- Filter (158) + : : : : +- Scan parquet (157) + : : : +- Exchange (165) + : : : +- Filter (164) + : : : +- Scan parquet (163) + : : +- Exchange (171) + : : +- Filter (170) + : : +- Scan parquet (169) + : +- Exchange (177) + : +- Filter (176) + : +- Scan parquet (175) + +- Exchange (184) + +- Project (183) + +- Filter (182) + +- Scan parquet (181) (1) Scan parquet @@ -166,746 +181,806 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [c_custkey#X, c_nationkey#X] Arguments: [c_custkey#X, c_nationkey#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: [n_nationkey#X, n_regionkey#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) NoopFilter +(96) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) NoopFilter +(113) NoopFilter Input [2]: [r_regionkey#X, r_name#X] Arguments: [r_regionkey#X, r_name#X] -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(127) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(128) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(129) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(130) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(131) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(132) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(133) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(134) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(135) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(136) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(137) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(138) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(139) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(140) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(141) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(142) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(143) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(144) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(145) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(146) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(147) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(148) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(134) Project +(149) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(150) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(151) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(152) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(153) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(154) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(140) Project +(155) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(156) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(157) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(158) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(159) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(160) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(146) Project +(161) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(162) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(163) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(164) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(165) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(166) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(152) Project +(167) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(168) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(169) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(170) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(171) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(172) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(158) Project +(173) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(174) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(175) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(176) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(177) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(178) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(164) Project +(179) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(180) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(181) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(182) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(183) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(184) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(185) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(171) Project +(186) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(187) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(188) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(189) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(175) Exchange +(190) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(191) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(192) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt index 5d1350564eac7..bbc0a70420239 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1-ras/spark34/9.txt @@ -1,114 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ NoopFilter (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ NoopFilter (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ NoopFilter (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ NoopFilter (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ NoopFilter (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ NoopFilter (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ NoopFilter (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ NoopFilter (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ NoopFilter (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ NoopFilter (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ NoopFilter (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ NoopFilter (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- ShuffledHashJoin Inner BuildRight (137) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (109) + : : : : : +- Project (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -130,558 +141,602 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) NoopFilter +(11) NoopFilter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) NoopFilter +(28) NoopFilter Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: [s_suppkey#X, s_nationkey#X] -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) NoopFilter +(45) NoopFilter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) NoopFilter +(62) NoopFilter Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: [o_orderkey#X, o_orderdate#X] -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) NoopFilter +(79) NoopFilter Input [2]: [n_nationkey#X, n_name#X] Arguments: [n_nationkey#X, n_name#X] -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(99) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(108) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(109) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(110) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(111) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(112) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(103) Project +(114) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(115) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(109) Project +(120) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(121) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(124) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(115) Project +(126) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(127) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(130) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(121) Project +(132) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(133) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(136) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(127) Project +(138) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(139) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(140) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(141) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(142) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(143) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt index 656f98574483b..1c8670c0fea4d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true)), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true)) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2), true) as decimal(26,4)))), DecimalType(38,6), true))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt index 313572f951add..8233c76692067 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/10.txt @@ -1,78 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- TakeOrderedAndProjectExecTransformer (65) + +- ^ ProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ShuffleQueryStage (59) + +- ColumnarExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -94,358 +100,382 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(57) ProjectExecTransformer +(63) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(65) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt index e91064e3580f9..bc664f216eb3b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/11.txt @@ -1,65 +1,70 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (77) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ FilterExecTransformer (46) + +- ^ RegularHashAggregateExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42) + +- ColumnarExchange (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (76) + +- Exchange (75) + +- Filter (74) + +- HashAggregate (73) + +- Exchange (72) + +- HashAggregate (71) + +- Project (70) + +- ShuffledHashJoin Inner BuildRight (69) + :- Exchange (64) + : +- Project (63) + : +- ShuffledHashJoin Inner BuildRight (62) + : :- Exchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- Exchange (61) + : +- Filter (60) + : +- Scan parquet (59) + +- Exchange (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) (1) Scan parquet @@ -81,292 +86,312 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(43) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(42) FilterExecTransformer +(46) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(48) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(49) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(51) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(53) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(56) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(58) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(59) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(61) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(62) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(63) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(64) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(67) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(68) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(70) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(71) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(72) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(73) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(cast(ps_availqty#X as decimal(10,0)) as decimal(12,2)))), DecimalType(23,2), true))#X AS value#X] -(69) Filter +(74) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(75) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(76) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(77) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt index 8bad82473a585..2004f617a1d89 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/12.txt @@ -1,46 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- ShuffledHashJoin Inner BuildLeft (45) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -62,196 +65,208 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(38) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(39) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(40) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(43) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(44) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(46) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(48) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(50) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(51) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt index 3268390701fcc..0e57911ad3ca3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftOuter BuildRight (46) + :- Exchange (41) + : +- Scan parquet (40) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -60,223 +63,235 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) FilterExecTransformer +(10) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(45) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(47) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt index dd4a53cfa8220..2bc0be8fcb672 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,144 +54,152 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [p_partkey#X, p_type#X] Arguments: isnotnull(p_partkey#X) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END)#X)), DecimalType(38,6), true)) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X as decimal(38,6)))), DecimalType(38,6), true) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt index c456274bfeaa4..40bb7d7d79838 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/15.txt @@ -1,44 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (45) +AdaptiveSparkPlan (47) +- == Final Plan == - VeloxColumnarToRowExec (30) - +- ^ SortExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (32) + +- ^ SortExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (44) - +- Exchange (43) - +- Project (42) - +- ShuffledHashJoin Inner BuildLeft (41) - :- Exchange (33) - : +- Filter (32) - : +- Scan parquet (31) - +- Filter (40) - +- HashAggregate (39) - +- Exchange (38) - +- HashAggregate (37) - +- Project (36) - +- Filter (35) - +- Scan parquet (34) + Sort (46) + +- Exchange (45) + +- Project (44) + +- ShuffledHashJoin Inner BuildLeft (43) + :- Exchange (35) + : +- Filter (34) + : +- Scan parquet (33) + +- Filter (42) + +- HashAggregate (41) + +- Exchange (40) + +- HashAggregate (39) + +- Project (38) + +- Filter (37) + +- Scan parquet (36) (1) Scan parquet @@ -60,182 +62,190 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(20) FilterExecTransformer +(21) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(26) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) SortExecTransformer +(30) SortExecTransformer Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(30) VeloxColumnarToRowExec +(32) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(31) Scan parquet +(33) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(33) Exchange +(35) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(34) Scan parquet +(36) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(35) Filter +(37) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(36) Project +(38) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(37) HashAggregate +(39) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(38) Exchange +(40) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) HashAggregate +(41) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS total_revenue#X] -(40) Filter +(42) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(41) ShuffledHashJoin +(43) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(42) Project +(44) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(43) Exchange +(45) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Sort +(46) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(45) AdaptiveSparkPlan +(47) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt index 7bbe4249fdcd1..90afc1a3db17c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/16.txt @@ -1,59 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (45) + +- ^ SortExecTransformer (43) + +- ^ InputIteratorTransformer (42) + +- ShuffleQueryStage (40) + +- ColumnarExchange (39) + +- VeloxAppendBatches (38) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- ^ ProjectExecTransformer (30) + +- ^ FlushableHashAggregateExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (66) + +- Exchange (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- ShuffledHashJoin Inner BuildRight (57) + :- Exchange (53) + : +- BroadcastHashJoin LeftAnti BuildRight (52) + : :- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -75,270 +78,282 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(29) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(38) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(39) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(41) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(43) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(48) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(49) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(50) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(51) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(53) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(56) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(58) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(59) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(62) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(63) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(66) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt index 248f638db9ad2..04753848cb48d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/17.txt @@ -1,56 +1,58 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ ProjectExecTransformer (34) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Project (48) + : +- ShuffledHashJoin Inner BuildRight (47) + : :- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -72,250 +74,258 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Arguments: isnotnull(l_partkey#X) -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(32) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(37) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(40) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(41) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(42) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(43) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(45) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(46) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7), true) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(56) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(58) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6), true) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt index 4281862189845..516601a5f4692 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/18.txt @@ -1,89 +1,94 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51) - +- ReusedExchange (50) + VeloxColumnarToRowExec (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ RegularHashAggregateExecTransformer (65) + +- ^ ProjectExecTransformer (64) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) + :- ^ InputIteratorTransformer (45) + : +- ShuffleQueryStage (43) + : +- ColumnarExchange (42) + : +- VeloxAppendBatches (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (37) + : +- ShuffleQueryStage (35) + : +- ColumnarExchange (34) + : +- VeloxAppendBatches (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ RegularHashAggregateExecTransformer (27) + : +- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + :- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52) + : +- ColumnarExchange (51) + : +- VeloxAppendBatches (50) + : +- ^ ProjectExecTransformer (48) + : +- ^ FilterExecTransformer (47) + : +- ^ Scan parquet (46) + +- ^ ProjectExecTransformer (61) + +- ^ FilterExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56) + +- ReusedExchange (55) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (101) + +- HashAggregate (100) + +- HashAggregate (99) + +- Project (98) + +- ShuffledHashJoin Inner BuildRight (97) + :- Exchange (86) + : +- Project (85) + : +- ShuffledHashJoin Inner BuildLeft (84) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (83) + : +- ShuffledHashJoin LeftSemi BuildRight (82) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Project (81) + : +- Filter (80) + : +- HashAggregate (79) + : +- Exchange (78) + : +- HashAggregate (77) + : +- Scan parquet (76) + +- ShuffledHashJoin LeftSemi BuildRight (96) + :- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Project (95) + +- Filter (94) + +- HashAggregate (93) + +- Exchange (92) + +- HashAggregate (91) + +- Scan parquet (90) (1) Scan parquet @@ -105,420 +110,440 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(36) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(38) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(41) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(42) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(43) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(44) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(45) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(50) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(51) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(52) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(53) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(54) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(55) ReusedExchange [Reuses operator id: 23] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(57) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(60) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(61) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(64) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(65) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(68) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(70) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(72) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(73) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(76) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(77) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(78) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(79) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(80) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(81) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(83) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(84) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(85) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(86) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(91) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(92) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(93) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(95) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(98) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(99) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(100) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(101) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt index 4ea9d09e13f75..d3d74c5ba7927 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,140 +53,148 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt index 4fe9560800868..df1ae98f903e5 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/20.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (126) +AdaptiveSparkPlan (136) +- == Final Plan == - VeloxColumnarToRowExec (86) - +- ^ SortExecTransformer (84) - +- ^ InputIteratorTransformer (83) - +- ShuffleQueryStage (81) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (96) + +- ^ SortExecTransformer (94) + +- ^ InputIteratorTransformer (93) + +- ShuffleQueryStage (91) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ FilterExecTransformer (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ FilterExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ FilterExecTransformer (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (125) - +- Exchange (124) - +- Project (123) - +- ShuffledHashJoin Inner BuildRight (122) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin LeftSemi BuildRight (115) - : :- Exchange (89) - : : +- Filter (88) - : : +- Scan parquet (87) - : +- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin Inner BuildLeft (112) - : :- Exchange (98) - : : +- ShuffledHashJoin LeftSemi BuildRight (97) - : : :- Exchange (92) - : : : +- Filter (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (111) - : +- Filter (110) - : +- HashAggregate (109) - : +- HashAggregate (108) - : +- ShuffledHashJoin LeftSemi BuildRight (107) - : :- Exchange (102) - : : +- Project (101) - : : +- Filter (100) - : : +- Scan parquet (99) - : +- Exchange (106) - : +- Project (105) - : +- Filter (104) - : +- Scan parquet (103) - +- Exchange (121) - +- Project (120) - +- Filter (119) - +- Scan parquet (118) + Sort (135) + +- Exchange (134) + +- Project (133) + +- ShuffledHashJoin Inner BuildRight (132) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin LeftSemi BuildRight (125) + : :- Exchange (99) + : : +- Filter (98) + : : +- Scan parquet (97) + : +- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin Inner BuildLeft (122) + : :- Exchange (108) + : : +- ShuffledHashJoin LeftSemi BuildRight (107) + : : :- Exchange (102) + : : : +- Filter (101) + : : : +- Scan parquet (100) + : : +- Exchange (106) + : : +- Project (105) + : : +- Filter (104) + : : +- Scan parquet (103) + : +- Exchange (121) + : +- Filter (120) + : +- HashAggregate (119) + : +- HashAggregate (118) + : +- ShuffledHashJoin LeftSemi BuildRight (117) + : :- Exchange (112) + : : +- Project (111) + : : +- Filter (110) + : : +- Scan parquet (109) + : +- Exchange (116) + : +- Project (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (131) + +- Project (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,518 +136,558 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) FilterExecTransformer +(37) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(78) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) InputAdapter +(92) InputAdapter Input [2]: [s_name#X, s_address#X] -(83) InputIteratorTransformer +(93) InputIteratorTransformer Input [2]: [s_name#X, s_address#X] -(84) SortExecTransformer +(94) SortExecTransformer Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(85) WholeStageCodegenTransformer (X) +(95) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(86) VeloxColumnarToRowExec +(96) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(87) Scan parquet +(97) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(89) Exchange +(99) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(91) Filter +(101) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(92) Exchange +(102) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(93) Scan parquet +(103) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(94) Filter +(104) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(95) Project +(105) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(96) Exchange +(106) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(98) Exchange +(108) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(109) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(100) Filter +(110) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(101) Project +(111) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(102) Exchange +(112) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) Scan parquet +(113) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(104) Filter +(114) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(105) Project +(115) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(106) Exchange +(116) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(107) ShuffledHashJoin +(117) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(108) HashAggregate +(118) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(109) HashAggregate +(119) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3), true) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(110) Filter +(120) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(111) Exchange +(121) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(cast(ps_availqty#X as decimal(10,0)) as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(113) Project +(123) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(114) Exchange +(124) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(116) Project +(126) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(117) Exchange +(127) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(120) Project +(130) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(121) Exchange +(131) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) ShuffledHashJoin +(132) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(123) Project +(133) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(124) Exchange +(134) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) Sort +(135) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(126) AdaptiveSparkPlan +(136) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt index 6f94bd5c8d011..664bb4e889056 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/21.txt @@ -1,104 +1,113 @@ == Physical Plan == -AdaptiveSparkPlan (119) +AdaptiveSparkPlan (128) +- == Final Plan == - VeloxColumnarToRowExec (82) - +- TakeOrderedAndProjectExecTransformer (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (91) + +- TakeOrderedAndProjectExecTransformer (90) + +- ^ RegularHashAggregateExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85) + +- ColumnarExchange (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (118) - +- HashAggregate (117) - +- Exchange (116) - +- HashAggregate (115) - +- Project (114) - +- ShuffledHashJoin Inner BuildRight (113) - :- Exchange (108) - : +- Project (107) - : +- ShuffledHashJoin Inner BuildRight (106) - : :- Exchange (101) - : : +- Project (100) - : : +- ShuffledHashJoin Inner BuildLeft (99) - : : :- Exchange (85) - : : : +- Filter (84) - : : : +- Scan parquet (83) - : : +- Exchange (98) - : : +- ShuffledHashJoin LeftAnti BuildRight (97) - : : :- ShuffledHashJoin LeftSemi BuildRight (92) - : : : :- Exchange (89) - : : : : +- Project (88) - : : : : +- Filter (87) - : : : : +- Scan parquet (86) - : : : +- Exchange (91) - : : : +- Scan parquet (90) - : : +- Exchange (96) - : : +- Project (95) - : : +- Filter (94) - : : +- Scan parquet (93) - : +- Exchange (105) - : +- Project (104) - : +- Filter (103) - : +- Scan parquet (102) - +- Exchange (112) - +- Project (111) - +- Filter (110) - +- Scan parquet (109) + TakeOrderedAndProject (127) + +- HashAggregate (126) + +- Exchange (125) + +- HashAggregate (124) + +- Project (123) + +- ShuffledHashJoin Inner BuildRight (122) + :- Exchange (117) + : +- Project (116) + : +- ShuffledHashJoin Inner BuildRight (115) + : :- Exchange (110) + : : +- Project (109) + : : +- ShuffledHashJoin Inner BuildLeft (108) + : : :- Exchange (94) + : : : +- Filter (93) + : : : +- Scan parquet (92) + : : +- Exchange (107) + : : +- ShuffledHashJoin LeftAnti BuildRight (106) + : : :- ShuffledHashJoin LeftSemi BuildRight (101) + : : : :- Exchange (98) + : : : : +- Project (97) + : : : : +- Filter (96) + : : : : +- Scan parquet (95) + : : : +- Exchange (100) + : : : +- Scan parquet (99) + : : +- Exchange (105) + : : +- Project (104) + : : +- Filter (103) + : : +- Scan parquet (102) + : +- Exchange (114) + : +- Project (113) + : +- Filter (112) + : +- Scan parquet (111) + +- Exchange (121) + +- Project (120) + +- Filter (119) + +- Scan parquet (118) (1) Scan parquet @@ -120,494 +129,530 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) FilterExecTransformer +(29) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) FilterExecTransformer +(54) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) FilterExecTransformer +(71) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(85) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(86) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(87) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(88) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) TakeOrderedAndProjectExecTransformer +(90) TakeOrderedAndProjectExecTransformer Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X], 0 -(82) VeloxColumnarToRowExec +(91) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(83) Scan parquet +(92) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(84) Filter +(93) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(85) Exchange +(94) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(86) Scan parquet +(95) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(87) Filter +(96) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(88) Project +(97) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(89) Exchange +(98) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(99) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(91) Exchange +(100) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(92) ShuffledHashJoin +(101) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(93) Scan parquet +(102) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(94) Filter +(103) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(95) Project +(104) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(96) Exchange +(105) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(106) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(98) Exchange +(107) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) ShuffledHashJoin +(108) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(100) Project +(109) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(101) Exchange +(110) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) Scan parquet +(111) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(103) Filter +(112) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(104) Project +(113) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(105) Exchange +(114) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) ShuffledHashJoin +(115) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(107) Project +(116) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(108) Exchange +(117) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) Scan parquet +(118) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(110) Filter +(119) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(111) Project +(120) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(112) Exchange +(121) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(113) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(114) Project +(123) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(115) HashAggregate +(124) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(116) Exchange +(125) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) HashAggregate +(126) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(118) TakeOrderedAndProject +(127) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(119) AdaptiveSparkPlan +(128) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt index 77c3b584125b3..5ada352ba19c0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/22.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftAnti BuildRight (42) + :- Exchange (39) + : +- Filter (38) + : +- Scan parquet (37) + +- Exchange (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,182 +62,194 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(29) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(30) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(31) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(32) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(33) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(34) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(37) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(38) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(39) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(40) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(43) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(44) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(45) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(46) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(47) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(48) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt index 6e4f9178de197..c51701bd0840d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,244 +76,260 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt index d2e7a9cffbd6e..2f1dd53da1efa 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/4.txt @@ -1,47 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftSemi BuildRight (46) + :- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -63,200 +66,212 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(40) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(44) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(45) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(47) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(48) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(49) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(51) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(52) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt index aff82bdec961b..e20b31dd7907b 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/5.txt @@ -1,115 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- Filter (110) + : : : : +- Scan parquet (109) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -131,552 +142,596 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(99) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(109) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(110) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(111) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(114) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(115) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(118) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(120) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(121) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(124) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(126) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(127) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(130) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(132) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(133) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(136) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(137) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(139) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(140) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(141) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(142) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true))#X AS revenue#X] -(132) Exchange +(143) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(144) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt index b39d8c1b2aec2..41f69e74d1a03 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4), true))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt index f0650d65e2cf3..e7c2257969763 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/7.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70) - +- ReusedExchange (69) + VeloxColumnarToRowExec (100) + +- ^ SortExecTransformer (98) + +- ^ InputIteratorTransformer (97) + +- ShuffleQueryStage (95) + +- ColumnarExchange (94) + +- VeloxAppendBatches (93) + +- ^ RegularHashAggregateExecTransformer (91) + +- ^ InputIteratorTransformer (90) + +- ShuffleQueryStage (88) + +- ColumnarExchange (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- ShuffledHashJoin Inner BuildRight (131) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin Inner BuildRight (125) + : :- Exchange (121) + : : +- Project (120) + : : +- ShuffledHashJoin Inner BuildRight (119) + : : :- Exchange (115) + : : : +- Project (114) + : : : +- ShuffledHashJoin Inner BuildRight (113) + : : : :- Exchange (109) + : : : : +- Project (108) + : : : : +- ShuffledHashJoin Inner BuildLeft (107) + : : : : :- Exchange (103) + : : : : : +- Filter (102) + : : : : : +- Scan parquet (101) + : : : : +- Exchange (106) + : : : : +- Filter (105) + : : : : +- Scan parquet (104) + : : : +- Exchange (112) + : : : +- Filter (111) + : : : +- Scan parquet (110) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Exchange (124) + : +- Filter (123) + : +- Scan parquet (122) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,524 +136,564 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [o_orderkey#X, o_custkey#X] Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(88) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(89) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(90) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(91) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(92) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(93) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(94) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(95) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(96) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(97) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(98) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(99) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(100) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(101) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(102) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(103) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(104) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(105) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(106) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(108) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(114) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(115) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(116) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(117) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(118) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(120) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(121) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(122) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(123) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(124) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(126) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(127) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(130) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt index e2a1907549f07..de63f62ffc7e8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/8.txt @@ -1,150 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (192) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (140) + +- ^ SortExecTransformer (138) + +- ^ InputIteratorTransformer (137) + +- ShuffleQueryStage (135) + +- ColumnarExchange (134) + +- VeloxAppendBatches (133) + +- ^ ProjectExecTransformer (131) + +- ^ RegularHashAggregateExecTransformer (130) + +- ^ InputIteratorTransformer (129) + +- ShuffleQueryStage (127) + +- ColumnarExchange (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (191) + +- Exchange (190) + +- HashAggregate (189) + +- Exchange (188) + +- HashAggregate (187) + +- Project (186) + +- ShuffledHashJoin Inner BuildRight (185) + :- Exchange (180) + : +- Project (179) + : +- ShuffledHashJoin Inner BuildRight (178) + : :- Exchange (174) + : : +- Project (173) + : : +- ShuffledHashJoin Inner BuildRight (172) + : : :- Exchange (168) + : : : +- Project (167) + : : : +- ShuffledHashJoin Inner BuildRight (166) + : : : :- Exchange (162) + : : : : +- Project (161) + : : : : +- ShuffledHashJoin Inner BuildRight (160) + : : : : :- Exchange (156) + : : : : : +- Project (155) + : : : : : +- ShuffledHashJoin Inner BuildRight (154) + : : : : : :- Exchange (150) + : : : : : : +- Project (149) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) + : : : : : : :- Exchange (144) + : : : : : : : +- Project (143) + : : : : : : : +- Filter (142) + : : : : : : : +- Scan parquet (141) + : : : : : : +- Exchange (147) + : : : : : : +- Filter (146) + : : : : : : +- Scan parquet (145) + : : : : : +- Exchange (153) + : : : : : +- Filter (152) + : : : : : +- Scan parquet (151) + : : : : +- Exchange (159) + : : : : +- Filter (158) + : : : : +- Scan parquet (157) + : : : +- Exchange (165) + : : : +- Filter (164) + : : : +- Scan parquet (163) + : : +- Exchange (171) + : : +- Filter (170) + : : +- Scan parquet (169) + : +- Exchange (177) + : +- Filter (176) + : +- Scan parquet (175) + +- Exchange (184) + +- Project (183) + +- Filter (182) + +- Scan parquet (181) (1) Scan parquet @@ -166,732 +181,792 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) FilterExecTransformer +(96) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) FilterExecTransformer +(113) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(127) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(128) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(129) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(130) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(131) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(132) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(133) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(134) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(135) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(136) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(137) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(138) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(139) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(140) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(141) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(142) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(143) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(144) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(145) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(146) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(147) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(148) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(149) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(150) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(151) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(152) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(153) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(154) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(155) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(156) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(157) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(158) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(159) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(160) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(161) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(162) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(163) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(164) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(165) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(166) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(167) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(168) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(169) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(170) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(171) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(172) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(173) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(174) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(175) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(176) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(177) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(178) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(179) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(180) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(181) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(182) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(183) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(184) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(185) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(186) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(187) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(188) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(189) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6), true) AS mkt_share#X] -(175) Exchange +(190) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(191) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(192) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt index b86e86d560e59..1fc81ef08365c 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark32/9.txt @@ -1,114 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- ShuffledHashJoin Inner BuildRight (137) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (109) + : : : : : +- Project (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -130,548 +141,592 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: isnotnull(o_orderkey#X) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(99) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(108) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(109) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(110) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(111) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(112) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(114) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(115) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(120) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(121) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(124) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(126) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(127) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(130) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(132) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(133) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(136) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(138) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2), true))), DecimalType(26,4), true) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4), true) as decimal(27,4)))), DecimalType(27,4), true) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(139) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(140) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(141) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(142) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(143) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt index 89de3133895b1..2f3257bbdafc8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), partial_sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))), sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS sum_disc_price#X, sum(CheckOverflow((promote_precision(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4))) * promote_precision(cast(CheckOverflow((1.00 + promote_precision(cast(l_tax#X as decimal(13,2)))), DecimalType(13,2)) as decimal(26,4)))), DecimalType(38,6)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt index 680971de4c1d8..8ca3c5d087a84 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/10.txt @@ -1,78 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- TakeOrderedAndProjectExecTransformer (65) + +- ^ ProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -94,358 +100,382 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(57) ProjectExecTransformer +(63) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(65) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(69) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(76) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(82) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [8]: [c_custkey#X, c_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt index b964ec1ed8d42..0e370c4f114af 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/11.txt @@ -1,65 +1,70 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (77) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ FilterExecTransformer (46) + +- ^ RegularHashAggregateExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (76) + +- Exchange (75) + +- Filter (74) + +- HashAggregate (73) + +- Exchange (72) + +- HashAggregate (71) + +- Project (70) + +- ShuffledHashJoin Inner BuildRight (69) + :- Exchange (64) + : +- Project (63) + : +- ShuffledHashJoin Inner BuildRight (62) + : :- Exchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- Exchange (61) + : +- Filter (60) + : +- Scan parquet (59) + +- Exchange (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) (1) Scan parquet @@ -81,549 +86,579 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(43) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(42) FilterExecTransformer +(46) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(48) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(49) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(51) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(53) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(56) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(58) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(59) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(61) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(62) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(58) Project +(63) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(64) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(67) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(68) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(65) Project +(70) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(71) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(72) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(73) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [2]: [ps_partkey#X, sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X AS value#X] -(69) Filter +(74) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(75) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(76) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(77) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - :- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ FilterExecTransformer (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (109) + +- ^ ProjectExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ RegularHashAggregateExecTransformer (105) + +- ^ ProjectExecTransformer (104) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (103) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- VeloxAppendBatches (94) + : +- ^ ProjectExecTransformer (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (91) + : :- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (90) + : +- ShuffleQueryStage (88), Statistics(X) + : +- ReusedExchange (87) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ReusedExchange (99) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (126) + +- HashAggregate (125) + +- Project (124) + +- ShuffledHashJoin Inner BuildRight (123) + :- Exchange (118) + : +- Project (117) + : +- ShuffledHashJoin Inner BuildRight (116) + : :- Exchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Exchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) + + +(78) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) FilterExecTransformer +(79) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(75) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(82) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(83) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(84) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(85) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(86) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(87) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(88) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(89) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(90) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(91) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(86) ProjectExecTransformer +(92) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(99) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(100) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(101) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(102) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(103) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(97) ProjectExecTransformer +(104) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(105) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(100) ProjectExecTransformer +(107) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] -(101) WholeStageCodegenTransformer (X) +(108) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(109) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(110) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(111) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(112) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(113) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(114) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(115) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(110) Project +(117) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(118) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(123) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(117) Project +(124) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(125) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(126) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X] Results [1]: [CheckOverflow((promote_precision(cast(sum(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(cast(ps_availqty#X as decimal(12,2)))), DecimalType(23,2)))#X as decimal(38,10))) * 0.0001000000), DecimalType(38,6)) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt index fabde3ecd6871..0ee1a20a6c391 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/12.txt @@ -1,46 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- ShuffledHashJoin Inner BuildLeft (45) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -62,196 +65,208 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(38) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(39) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(40) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(43) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(44) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(43) Project +(46) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(48) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(50) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(51) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt index 6ca6e75ac545e..be5442000bde0 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftOuter BuildRight (46) + :- Exchange (41) + : +- Scan parquet (40) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -60,223 +63,235 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) FilterExecTransformer +(10) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(45) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(44) Project +(47) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt index c1a4a2fe884fc..afac58cb52bc1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,144 +54,152 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [p_partkey#X, p_type#X] Arguments: isnotnull(p_partkey#X) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END), sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [CheckOverflow((promote_precision(CheckOverflow((100.0000 * promote_precision(sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END)#X)), DecimalType(38,6))) / promote_precision(cast(sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X as decimal(38,6)))), DecimalType(38,6)) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt index a45c614317829..7b33a6b69bd9e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (29) + +- AQEShuffleRead (28) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- Project (41) + +- ShuffledHashJoin Inner BuildLeft (40) + :- Exchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Filter (39) + +- HashAggregate (38) + +- Exchange (37) + +- HashAggregate (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -59,328 +61,336 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(20) FilterExecTransformer +(21) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(26) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(28) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(30) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(32) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(33) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(35) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(37) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(37) Filter +(39) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(40) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join condition: None -(39) Project +(41) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(42) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ FilterExecTransformer (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (48) + +- ^ ProjectExecTransformer (47) + +- ^ FilterExecTransformer (46) + +- ^ Scan parquet (45) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(43) Scan parquet +(45) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) FilterExecTransformer +(46) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(45) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(48) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(54) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS total_revenue#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt index 9a927b77805e0..287ae9f00ed13 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/16.txt @@ -1,59 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (45) + +- ^ SortExecTransformer (43) + +- ^ InputIteratorTransformer (42) + +- ShuffleQueryStage (40), Statistics(X) + +- ColumnarExchange (39) + +- VeloxAppendBatches (38) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ ProjectExecTransformer (30) + +- ^ FlushableHashAggregateExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (66) + +- Exchange (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- ShuffledHashJoin Inner BuildRight (57) + :- Exchange (53) + : +- BroadcastHashJoin LeftAnti BuildRight (52) + : :- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -75,270 +78,282 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(29) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(38) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(39) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(41) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(43) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(48) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(49) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(50) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(51) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(50) Exchange +(53) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(56) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(55) Project +(58) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(59) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(62) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(63) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(66) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt index 48ba7ddab0935..23d34ad9b587e 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/17.txt @@ -1,56 +1,58 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ ProjectExecTransformer (34) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Project (48) + : +- ShuffledHashJoin Inner BuildRight (47) + : :- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -72,250 +74,258 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Arguments: isnotnull(l_partkey#X) -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(32) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(37) ProjectExecTransformer Output [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(40) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(41) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(42) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(43) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(45) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(46) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(46) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [CheckOverflow((0.200000 * promote_precision(avg(l_quantity#X)#X)), DecimalType(18,7)) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(56) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(58) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [CheckOverflow((promote_precision(sum(l_extendedprice#X)#X) / 7.00), DecimalType(27,6)) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt index a15a185bbd779..fdf15018cafa2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/18.txt @@ -1,89 +1,94 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ RegularHashAggregateExecTransformer (65) + +- ^ ProjectExecTransformer (64) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) + :- ^ InputIteratorTransformer (45) + : +- ShuffleQueryStage (43), Statistics(X) + : +- ColumnarExchange (42) + : +- VeloxAppendBatches (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (37) + : +- ShuffleQueryStage (35), Statistics(X) + : +- ColumnarExchange (34) + : +- VeloxAppendBatches (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ RegularHashAggregateExecTransformer (27) + : +- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + :- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- VeloxAppendBatches (50) + : +- ^ ProjectExecTransformer (48) + : +- ^ FilterExecTransformer (47) + : +- ^ Scan parquet (46) + +- ^ ProjectExecTransformer (61) + +- ^ FilterExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ReusedExchange (55) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (101) + +- HashAggregate (100) + +- HashAggregate (99) + +- Project (98) + +- ShuffledHashJoin Inner BuildRight (97) + :- Exchange (86) + : +- Project (85) + : +- ShuffledHashJoin Inner BuildLeft (84) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (83) + : +- ShuffledHashJoin LeftSemi BuildRight (82) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Project (81) + : +- Filter (80) + : +- HashAggregate (79) + : +- Exchange (78) + : +- HashAggregate (77) + : +- Scan parquet (76) + +- ShuffledHashJoin LeftSemi BuildRight (96) + :- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Project (95) + +- Filter (94) + +- HashAggregate (93) + +- Exchange (92) + +- HashAggregate (91) + +- Scan parquet (90) (1) Scan parquet @@ -105,420 +110,440 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(36) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(38) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(36) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(41) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(42) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(43) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(44) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(45) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(50) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(51) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(52) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(53) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(54) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(55) ReusedExchange [Reuses operator id: 23] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(57) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(60) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(61) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(58) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(59) ProjectExecTransformer +(64) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(65) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(68) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(70) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(72) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(73) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(76) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(77) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(78) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(79) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(80) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(81) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(78) Exchange +(83) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(84) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(80) Project +(85) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(86) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(91) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(92) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(93) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(95) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(93) Project +(98) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(99) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(100) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(101) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt index 80f0189d5d653..14a5515d1e796 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,140 +53,148 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt index a0dcc7029a354..7b840720bc90f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/20.txt @@ -1,109 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (93) + +- AQEShuffleRead (92) + +- ShuffleQueryStage (91), Statistics(X) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74), Statistics(X) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ FilterExecTransformer (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58), Statistics(X) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ FilterExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ FilterExecTransformer (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (132) + +- Exchange (131) + +- Project (130) + +- ShuffledHashJoin Inner BuildRight (129) + :- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin LeftSemi BuildRight (122) + : :- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (121) + : +- Project (120) + : +- ShuffledHashJoin Inner BuildLeft (119) + : :- Exchange (105) + : : +- ShuffledHashJoin LeftSemi BuildRight (104) + : : :- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Exchange (118) + : +- Filter (117) + : +- HashAggregate (116) + : +- HashAggregate (115) + : +- ShuffledHashJoin LeftSemi BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (128) + +- Project (127) + +- Filter (126) + +- Scan parquet (125) (1) Scan parquet @@ -125,508 +135,548 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) FilterExecTransformer +(37) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(78) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(92) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(93) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(94) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(95) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(96) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(97) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(99) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(95) Exchange +(105) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(106) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(107) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(108) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(112) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(113) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join condition: None -(105) HashAggregate +(115) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(116) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [CheckOverflow((0.50 * promote_precision(sum(l_quantity#X)#X)), DecimalType(24,3)) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(117) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(118) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join condition: None -(113) Project +(123) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(124) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(125) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(126) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(127) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(128) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(129) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(120) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt index 3635363cfe476..245faea0cde80 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/21.txt @@ -1,103 +1,112 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (90) + +- ^ RegularHashAggregateExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- ShuffledHashJoin Inner BuildLeft (107) + : : :- Exchange (93) + : : : +- Filter (92) + : : : +- Scan parquet (91) + : : +- Exchange (106) + : : +- ShuffledHashJoin LeftAnti BuildRight (105) + : : :- ShuffledHashJoin LeftSemi BuildRight (100) + : : : :- Exchange (97) + : : : : +- Project (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (99) + : : : +- Scan parquet (98) + : : +- Exchange (104) + : : +- Project (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -119,490 +128,526 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) FilterExecTransformer +(29) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) FilterExecTransformer +(54) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) FilterExecTransformer +(71) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(85) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(86) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(87) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(88) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(91) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(92) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(93) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(94) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(95) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(96) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(98) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(101) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(102) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(103) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(104) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(106) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(99) Project +(108) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(109) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(111) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(112) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(113) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(106) Project +(115) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(116) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(117) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(118) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(119) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(120) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(113) Project +(122) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(123) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(124) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(125) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(126) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt index 3c18ab436ed29..2a822968008a1 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/22.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftAnti BuildRight (42) + :- Exchange (39) + : +- Filter (38) + : +- Scan parquet (37) + +- Exchange (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,316 +62,328 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(29) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(30) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(31) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(32) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(33) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(34) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(37) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(38) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(39) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(40) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(40) Project +(43) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(44) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(45) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(46) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(47) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(48) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (61) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- ^ FlushableHashAggregateExecTransformer (53) + +- ^ ProjectExecTransformer (52) + +- ^ FilterExecTransformer (51) + +- ^ Scan parquet (50) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (67) + +- Exchange (66) + +- HashAggregate (65) + +- Project (64) + +- Filter (63) + +- Scan parquet (62) -(47) Scan parquet +(50) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) FilterExecTransformer +(51) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(49) ProjectExecTransformer +(52) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(53) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(55) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(56) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(57) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(58) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(62) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(63) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(64) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(66) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(67) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (61) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- ^ FlushableHashAggregateExecTransformer (53) + +- ^ ProjectExecTransformer (52) + +- ^ FilterExecTransformer (51) + +- ^ Scan parquet (50) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) \ No newline at end of file + HashAggregate (67) + +- Exchange (66) + +- HashAggregate (65) + +- Project (64) + +- Filter (63) + +- Scan parquet (62) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt index eebd274d47093..51408f03e4e7d 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,244 +76,260 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [4]: [l_orderkey#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt index f05c9a5378c6f..deba9b23f1fa3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/4.txt @@ -1,47 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftSemi BuildRight (46) + :- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -63,200 +66,212 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(40) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(44) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(45) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(44) Project +(47) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(48) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(49) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(51) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(52) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt index 027ea4a926d2a..257c7a998c7ea 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/5.txt @@ -1,115 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- Filter (110) + : : : : +- Scan parquet (109) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -131,552 +142,596 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(99) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(109) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(110) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(111) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join condition: None -(103) Project +(114) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(115) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(118) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join condition: None -(109) Project +(120) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(121) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(124) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join condition: None -(115) Project +(126) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(127) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(130) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(121) Project +(132) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(133) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(136) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(137) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(128) Project +(139) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(140) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(141) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(142) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X] Results [2]: [n_name#X, sum(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)))#X AS revenue#X] -(132) Exchange +(143) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(144) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt index 68854bdea4737..3a3148c1c1133 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))] Aggregate Attributes [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X] Results [1]: [sum(CheckOverflow((promote_precision(l_extendedprice#X) * promote_precision(l_discount#X)), DecimalType(25,4)))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt index eb5979fb5d847..1255a3a0c6ec4 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/7.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (100) + +- ^ SortExecTransformer (98) + +- ^ InputIteratorTransformer (97) + +- ShuffleQueryStage (95), Statistics(X) + +- ColumnarExchange (94) + +- VeloxAppendBatches (93) + +- ^ RegularHashAggregateExecTransformer (91) + +- ^ InputIteratorTransformer (90) + +- ShuffleQueryStage (88), Statistics(X) + +- ColumnarExchange (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- ShuffledHashJoin Inner BuildRight (131) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin Inner BuildRight (125) + : :- Exchange (121) + : : +- Project (120) + : : +- ShuffledHashJoin Inner BuildRight (119) + : : :- Exchange (115) + : : : +- Project (114) + : : : +- ShuffledHashJoin Inner BuildRight (113) + : : : :- Exchange (109) + : : : : +- Project (108) + : : : : +- ShuffledHashJoin Inner BuildLeft (107) + : : : : :- Exchange (103) + : : : : : +- Filter (102) + : : : : : +- Scan parquet (101) + : : : : +- Exchange (106) + : : : : +- Filter (105) + : : : : +- Scan parquet (104) + : : : +- Exchange (112) + : : : +- Filter (111) + : : : +- Scan parquet (110) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Exchange (124) + : +- Filter (123) + : +- Scan parquet (122) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,524 +136,564 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [o_orderkey#X, o_custkey#X] Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(88) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(89) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(90) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(91) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(92) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(93) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(94) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(95) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(96) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(97) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(98) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(99) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(100) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(101) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(102) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(103) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(104) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(105) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(106) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join condition: None -(98) Project +(108) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(104) Project +(114) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(115) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(116) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(117) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(118) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(110) Project +(120) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(121) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(122) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(123) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(124) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(116) Project +(126) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(127) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(130) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt index 98bba133502c8..8c66866aa476f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/8.txt @@ -1,150 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (192) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (140) + +- ^ SortExecTransformer (138) + +- ^ InputIteratorTransformer (137) + +- ShuffleQueryStage (135), Statistics(X) + +- ColumnarExchange (134) + +- VeloxAppendBatches (133) + +- ^ ProjectExecTransformer (131) + +- ^ RegularHashAggregateExecTransformer (130) + +- ^ InputIteratorTransformer (129) + +- ShuffleQueryStage (127), Statistics(X) + +- ColumnarExchange (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (191) + +- Exchange (190) + +- HashAggregate (189) + +- Exchange (188) + +- HashAggregate (187) + +- Project (186) + +- ShuffledHashJoin Inner BuildRight (185) + :- Exchange (180) + : +- Project (179) + : +- ShuffledHashJoin Inner BuildRight (178) + : :- Exchange (174) + : : +- Project (173) + : : +- ShuffledHashJoin Inner BuildRight (172) + : : :- Exchange (168) + : : : +- Project (167) + : : : +- ShuffledHashJoin Inner BuildRight (166) + : : : :- Exchange (162) + : : : : +- Project (161) + : : : : +- ShuffledHashJoin Inner BuildRight (160) + : : : : :- Exchange (156) + : : : : : +- Project (155) + : : : : : +- ShuffledHashJoin Inner BuildRight (154) + : : : : : :- Exchange (150) + : : : : : : +- Project (149) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) + : : : : : : :- Exchange (144) + : : : : : : : +- Project (143) + : : : : : : : +- Filter (142) + : : : : : : : +- Scan parquet (141) + : : : : : : +- Exchange (147) + : : : : : : +- Filter (146) + : : : : : : +- Scan parquet (145) + : : : : : +- Exchange (153) + : : : : : +- Filter (152) + : : : : : +- Scan parquet (151) + : : : : +- Exchange (159) + : : : : +- Filter (158) + : : : : +- Scan parquet (157) + : : : +- Exchange (165) + : : : +- Filter (164) + : : : +- Scan parquet (163) + : : +- Exchange (171) + : : +- Filter (170) + : : +- Scan parquet (169) + : +- Exchange (177) + : +- Filter (176) + : +- Scan parquet (175) + +- Exchange (184) + +- Project (183) + +- Filter (182) + +- Scan parquet (181) (1) Scan parquet @@ -166,732 +181,792 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) FilterExecTransformer +(96) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) FilterExecTransformer +(113) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(127) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(128) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(129) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(130) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(131) ProjectExecTransformer Output [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(132) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(133) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(134) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(135) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(136) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(137) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(138) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(139) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(140) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(141) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(142) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(143) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(144) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(145) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(146) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(147) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(148) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(134) Project +(149) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(150) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(151) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(152) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(153) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(154) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(140) Project +(155) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(156) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(157) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(158) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(159) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(160) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(146) Project +(161) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(162) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(163) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(164) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(165) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(166) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join condition: None -(152) Project +(167) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(168) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(169) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(170) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(171) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(172) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(158) Project +(173) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(174) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(175) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(176) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(177) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(178) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(164) Project +(179) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(180) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(181) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(182) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(183) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(184) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(185) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join condition: None -(171) Project +(186) Project Output [3]: [year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(187) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(188) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(189) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, CheckOverflow((promote_precision(sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X) / promote_precision(sum(volume#X)#X)), DecimalType(38,6)) AS mkt_share#X] -(175) Exchange +(190) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(191) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(192) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt index 7f3917f80457c..7435043d769db 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark33/9.txt @@ -1,114 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- ShuffledHashJoin Inner BuildRight (137) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (109) + : : : : : +- Project (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -130,548 +141,592 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: isnotnull(o_orderkey#X) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(99) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(108) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(109) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(110) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(111) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(112) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join condition: None -(103) Project +(114) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(115) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join condition: None -(109) Project +(120) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(121) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(124) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join condition: None -(115) Project +(126) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(127) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(130) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join condition: None -(121) Project +(132) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(133) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(136) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join condition: None -(127) Project +(138) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, CheckOverflow((promote_precision(cast(CheckOverflow((promote_precision(cast(l_extendedprice#X as decimal(13,2))) * promote_precision(CheckOverflow((1.00 - promote_precision(cast(l_discount#X as decimal(13,2)))), DecimalType(13,2)))), DecimalType(26,4)) as decimal(27,4))) - promote_precision(cast(CheckOverflow((promote_precision(ps_supplycost#X) * promote_precision(l_quantity#X)), DecimalType(25,4)) as decimal(27,4)))), DecimalType(27,4)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(139) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(140) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(141) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(142) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(143) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt index 090a9522f13a8..468ad7c5d3aab 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/1.txt @@ -1,29 +1,30 @@ == Physical Plan == -AdaptiveSparkPlan (28) +AdaptiveSparkPlan (29) +- == Final Plan == - VeloxColumnarToRowExec (19) - +- ^ SortExecTransformer (17) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ RegularHashAggregateExecTransformer (11) - +- ^ InputIteratorTransformer (10) - +- ShuffleQueryStage (8), Statistics(X) - +- ColumnarExchange (7) - +- ^ ProjectExecTransformer (5) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (20) + +- ^ SortExecTransformer (18) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- ^ ProjectExecTransformer (5) + +- ^ FlushableHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - Sort (27) - +- Exchange (26) - +- HashAggregate (25) - +- Exchange (24) - +- HashAggregate (23) - +- Project (22) - +- Filter (21) - +- Scan parquet (20) + Sort (28) + +- Exchange (27) + +- HashAggregate (26) + +- Exchange (25) + +- HashAggregate (24) + +- Project (23) + +- Filter (22) + +- Scan parquet (21) (1) Scan parquet @@ -81,72 +82,76 @@ Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(13) ColumnarExchange +(13) VeloxAppendBatches +Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] +Arguments: X + +(14) ColumnarExchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(15) ShuffleQueryStage Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: X -(15) InputAdapter +(16) InputAdapter Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(16) InputIteratorTransformer +(17) InputIteratorTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(17) SortExecTransformer +(18) SortExecTransformer Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(18) WholeStageCodegenTransformer (X) +(19) WholeStageCodegenTransformer (X) Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: false -(19) VeloxColumnarToRowExec +(20) VeloxColumnarToRowExec Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] -(20) Scan parquet +(21) Scan parquet Output [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), LessThanOrEqual(l_shipdate,1998-09-02)] ReadSchema: struct -(21) Filter +(22) Filter Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] Condition : (isnotnull(l_shipdate#X) AND (l_shipdate#X <= 1998-09-02)) -(22) Project +(23) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Input [7]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X, l_shipdate#X] -(23) HashAggregate +(24) HashAggregate Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_tax#X, l_returnflag#X, l_linestatus#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [partial_sum(l_quantity#X), partial_sum(l_extendedprice#X), partial_sum((l_extendedprice#X * (1 - l_discount#X))), partial_sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), partial_avg(l_quantity#X), partial_avg(l_extendedprice#X), partial_avg(l_discount#X), partial_count(1)] Aggregate Attributes [15]: [sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Results [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] -(24) Exchange +(25) Exchange Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Arguments: hashpartitioning(l_returnflag#X, l_linestatus#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(25) HashAggregate +(26) HashAggregate Input [17]: [l_returnflag#X, l_linestatus#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, isEmpty#X, sum#X, count#X, sum#X, count#X, sum#X, count#X, count#X] Keys [2]: [l_returnflag#X, l_linestatus#X] Functions [8]: [sum(l_quantity#X), sum(l_extendedprice#X), sum((l_extendedprice#X * (1 - l_discount#X))), sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X))), avg(l_quantity#X), avg(l_extendedprice#X), avg(l_discount#X), count(1)] Aggregate Attributes [8]: [sum(l_quantity#X)#X, sum(l_extendedprice#X)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X, avg(l_quantity#X)#X, avg(l_extendedprice#X)#X, avg(l_discount#X)#X, count(1)#X] Results [10]: [l_returnflag#X, l_linestatus#X, sum(l_quantity#X)#X AS sum_qty#X, sum(l_extendedprice#X)#X AS sum_base_price#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS sum_disc_price#X, sum(((l_extendedprice#X * (1 - l_discount#X)) * (1 + l_tax#X)))#X AS sum_charge#X, avg(l_quantity#X)#X AS avg_qty#X, avg(l_extendedprice#X)#X AS avg_price#X, avg(l_discount#X)#X AS avg_disc#X, count(1)#X AS count_order#X] -(26) Exchange +(27) Exchange Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: rangepartitioning(l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Sort +(28) Sort Input [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: [l_returnflag#X ASC NULLS FIRST, l_linestatus#X ASC NULLS FIRST], true, 0 -(28) AdaptiveSparkPlan +(29) AdaptiveSparkPlan Output [10]: [l_returnflag#X, l_linestatus#X, sum_qty#X, sum_base_price#X, sum_disc_price#X, sum_charge#X, avg_qty#X, avg_price#X, avg_disc#X, count_order#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt index 02cf374dd013c..c740b49a1a8b8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/10.txt @@ -1,78 +1,84 @@ == Physical Plan == -AdaptiveSparkPlan (87) +AdaptiveSparkPlan (93) +- == Final Plan == - VeloxColumnarToRowExec (60) - +- TakeOrderedAndProjectExecTransformer (59) - +- ^ ProjectExecTransformer (57) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ ProjectExecTransformer (50) - +- ^ FlushableHashAggregateExecTransformer (49) - +- ^ ProjectExecTransformer (48) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - :- ^ InputIteratorTransformer (38) - : +- ShuffleQueryStage (36), Statistics(X) - : +- ColumnarExchange (35) - : +- ^ ProjectExecTransformer (33) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : :- ^ InputIteratorTransformer (23) - : : +- ShuffleQueryStage (21), Statistics(X) - : : +- ColumnarExchange (20) - : : +- ^ ProjectExecTransformer (18) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ InputIteratorTransformer (31) - : +- ShuffleQueryStage (29), Statistics(X) - : +- ColumnarExchange (28) - : +- ^ ProjectExecTransformer (26) - : +- ^ FilterExecTransformer (25) - : +- ^ Scan parquet (24) - +- ^ InputIteratorTransformer (46) - +- ShuffleQueryStage (44), Statistics(X) - +- ColumnarExchange (43) - +- ^ ProjectExecTransformer (41) - +- ^ FilterExecTransformer (40) - +- ^ Scan parquet (39) + VeloxColumnarToRowExec (66) + +- TakeOrderedAndProjectExecTransformer (65) + +- ^ ProjectExecTransformer (63) + +- ^ RegularHashAggregateExecTransformer (62) + +- ^ InputIteratorTransformer (61) + +- ShuffleQueryStage (59), Statistics(X) + +- ColumnarExchange (58) + +- ^ ProjectExecTransformer (56) + +- ^ FlushableHashAggregateExecTransformer (55) + +- ^ ProjectExecTransformer (54) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + :- ^ InputIteratorTransformer (43) + : +- ShuffleQueryStage (41), Statistics(X) + : +- ColumnarExchange (40) + : +- VeloxAppendBatches (39) + : +- ^ ProjectExecTransformer (37) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : :- ^ InputIteratorTransformer (26) + : : +- ShuffleQueryStage (24), Statistics(X) + : : +- ColumnarExchange (23) + : : +- VeloxAppendBatches (22) + : : +- ^ ProjectExecTransformer (20) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ InputIteratorTransformer (35) + : +- ShuffleQueryStage (33), Statistics(X) + : +- ColumnarExchange (32) + : +- VeloxAppendBatches (31) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ Scan parquet (27) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ ProjectExecTransformer (46) + +- ^ FilterExecTransformer (45) + +- ^ Scan parquet (44) +- == Initial Plan == - TakeOrderedAndProject (86) - +- HashAggregate (85) - +- Exchange (84) - +- HashAggregate (83) - +- Project (82) - +- ShuffledHashJoin Inner BuildRight (81) - :- Exchange (77) - : +- Project (76) - : +- ShuffledHashJoin Inner BuildRight (75) - : :- Exchange (70) - : : +- Project (69) - : : +- ShuffledHashJoin Inner BuildRight (68) - : : :- Exchange (63) - : : : +- Filter (62) - : : : +- Scan parquet (61) - : : +- Exchange (67) - : : +- Project (66) - : : +- Filter (65) - : : +- Scan parquet (64) - : +- Exchange (74) - : +- Project (73) - : +- Filter (72) - : +- Scan parquet (71) - +- Exchange (80) - +- Filter (79) - +- Scan parquet (78) + TakeOrderedAndProject (92) + +- HashAggregate (91) + +- Exchange (90) + +- HashAggregate (89) + +- Project (88) + +- ShuffledHashJoin Inner BuildRight (87) + :- Exchange (83) + : +- Project (82) + : +- ShuffledHashJoin Inner BuildRight (81) + : :- Exchange (76) + : : +- Project (75) + : : +- ShuffledHashJoin Inner BuildRight (74) + : : :- Exchange (69) + : : : +- Filter (68) + : : : +- Scan parquet (67) + : : +- Exchange (73) + : : +- Project (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (80) + : +- Project (79) + : +- Filter (78) + : +- Scan parquet (77) + +- Exchange (86) + +- Filter (85) + +- Scan parquet (84) (1) Scan parquet @@ -94,364 +100,388 @@ Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acct Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] +Arguments: X + +(6) ColumnarExchange Input [8]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [9]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [9]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Arguments: ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [10]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(49) FlushableHashAggregateExecTransformer +(55) FlushableHashAggregateExecTransformer Input [10]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(50) ProjectExecTransformer +(56) ProjectExecTransformer Output [10]: [hash(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(51) WholeStageCodegenTransformer (X) +(57) WholeStageCodegenTransformer (X) Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: false -(52) ColumnarExchange +(58) ColumnarExchange Input [10]: [hash_partition_key#X, c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(59) ShuffleQueryStage Output [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: X -(54) InputAdapter +(60) InputAdapter Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(55) InputIteratorTransformer +(61) InputIteratorTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(56) RegularHashAggregateExecTransformer +(62) RegularHashAggregateExecTransformer Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(57) ProjectExecTransformer +(63) ProjectExecTransformer Output [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Input [8]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(58) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: false -(59) TakeOrderedAndProjectExecTransformer +(65) TakeOrderedAndProjectExecTransformer Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X], 0 -(60) VeloxColumnarToRowExec +(66) VeloxColumnarToRowExec Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(61) Scan parquet +(67) Scan parquet Output [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(62) Filter +(68) Filter Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(63) Exchange +(69) Exchange Input [7]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) Scan parquet +(70) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-10-01), LessThan(o_orderdate,1994-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(65) Filter +(71) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-10-01)) AND (o_orderdate#X < 1994-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(66) Project +(72) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(67) Exchange +(73) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) ShuffledHashJoin +(74) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(69) Project +(75) Project Output [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, o_custkey#X] -(70) Exchange +(76) Exchange Input [8]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(77) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_returnflag), EqualTo(l_returnflag,R), IsNotNull(l_orderkey)] ReadSchema: struct -(72) Filter +(78) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] Condition : ((isnotnull(l_returnflag#X) AND (l_returnflag#X = R)) AND isnotnull(l_orderkey#X)) -(73) Project +(79) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_returnflag#X] -(74) Exchange +(80) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(75) ShuffledHashJoin +(81) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(76) Project +(82) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, o_orderkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(77) Exchange +(83) Exchange Input [9]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(78) Scan parquet +(84) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(79) Filter +(85) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(80) Exchange +(86) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(81) ShuffledHashJoin +(87) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(82) Project +(88) Project Output [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Input [11]: [c_custkey#X, c_name#X, c_address#X, c_nationkey#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_nationkey#X, n_name#X] -(83) HashAggregate +(89) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_address#X, c_phone#X, c_acctbal#X, c_comment#X, l_extendedprice#X, l_discount#X, n_name#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] -(84) Exchange +(90) Exchange Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Arguments: hashpartitioning(c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) HashAggregate +(91) HashAggregate Input [9]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X, sum#X, isEmpty#X] Keys [7]: [c_custkey#X, c_name#X, c_acctbal#X, c_phone#X, n_name#X, c_address#X, c_comment#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [8]: [c_custkey#X, c_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(86) TakeOrderedAndProject +(92) TakeOrderedAndProject Input [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: X, [revenue#X DESC NULLS LAST], [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] -(87) AdaptiveSparkPlan +(93) AdaptiveSparkPlan Output [8]: [c_custkey#X, c_name#X, revenue#X, c_acctbal#X, n_name#X, c_address#X, c_phone#X, c_comment#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt index b14866370aab5..89dd19ffe0ce2 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/11.txt @@ -1,65 +1,70 @@ == Physical Plan == -AdaptiveSparkPlan (72) +AdaptiveSparkPlan (77) +- == Final Plan == - VeloxColumnarToRowExec (50) - +- ^ SortExecTransformer (48) - +- ^ InputIteratorTransformer (47) - +- ShuffleQueryStage (45), Statistics(X) - +- ColumnarExchange (44) - +- ^ FilterExecTransformer (42) - +- ^ RegularHashAggregateExecTransformer (41) - +- ^ InputIteratorTransformer (40) - +- ShuffleQueryStage (38), Statistics(X) - +- ColumnarExchange (37) - +- ^ ProjectExecTransformer (35) - +- ^ FlushableHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (55) + +- ^ SortExecTransformer (53) + +- ^ InputIteratorTransformer (52) + +- ShuffleQueryStage (50), Statistics(X) + +- ColumnarExchange (49) + +- VeloxAppendBatches (48) + +- ^ FilterExecTransformer (46) + +- ^ RegularHashAggregateExecTransformer (45) + +- ^ InputIteratorTransformer (44) + +- ShuffleQueryStage (42), Statistics(X) + +- ColumnarExchange (41) + +- ^ ProjectExecTransformer (39) + +- ^ FlushableHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - Sort (71) - +- Exchange (70) - +- Filter (69) - +- HashAggregate (68) - +- Exchange (67) - +- HashAggregate (66) - +- Project (65) - +- ShuffledHashJoin Inner BuildRight (64) - :- Exchange (59) - : +- Project (58) - : +- ShuffledHashJoin Inner BuildRight (57) - : :- Exchange (53) - : : +- Filter (52) - : : +- Scan parquet (51) - : +- Exchange (56) - : +- Filter (55) - : +- Scan parquet (54) - +- Exchange (63) - +- Project (62) - +- Filter (61) - +- Scan parquet (60) + Sort (76) + +- Exchange (75) + +- Filter (74) + +- HashAggregate (73) + +- Exchange (72) + +- HashAggregate (71) + +- Project (70) + +- ShuffledHashJoin Inner BuildRight (69) + :- Exchange (64) + : +- Project (63) + : +- ShuffledHashJoin Inner BuildRight (62) + : :- Exchange (58) + : : +- Filter (57) + : : +- Scan parquet (56) + : +- Exchange (61) + : +- Filter (60) + : +- Scan parquet (59) + +- Exchange (68) + +- Project (67) + +- Filter (66) + +- Scan parquet (65) (1) Scan parquet @@ -81,557 +86,587 @@ Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [1]: [n_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [1]: [n_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(34) FlushableHashAggregateExecTransformer +(38) FlushableHashAggregateExecTransformer Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(35) ProjectExecTransformer +(39) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(36) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: false -(37) ColumnarExchange +(41) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(38) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: X -(39) InputAdapter +(43) InputAdapter Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(40) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] -(41) RegularHashAggregateExecTransformer +(45) RegularHashAggregateExecTransformer Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(42) FilterExecTransformer +(46) FilterExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(43) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(44) ColumnarExchange +(48) VeloxAppendBatches +Input [2]: [ps_partkey#X, value#X] +Arguments: X + +(49) ColumnarExchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(45) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [ps_partkey#X, value#X] Arguments: X -(46) InputAdapter +(51) InputAdapter Input [2]: [ps_partkey#X, value#X] -(47) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [ps_partkey#X, value#X] -(48) SortExecTransformer +(53) SortExecTransformer Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(49) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [ps_partkey#X, value#X] Arguments: false -(50) VeloxColumnarToRowExec +(55) VeloxColumnarToRowExec Input [2]: [ps_partkey#X, value#X] -(51) Scan parquet +(56) Scan parquet Output [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(52) Filter +(57) Filter Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(53) Exchange +(58) Exchange Input [4]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) Scan parquet +(59) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(55) Filter +(60) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(56) Exchange +(61) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(57) ShuffledHashJoin +(62) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(58) Project +(63) Project Output [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(59) Exchange +(64) Exchange Input [4]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(60) Scan parquet +(65) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(61) Filter +(66) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(62) Project +(67) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(63) Exchange +(68) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) ShuffledHashJoin +(69) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(65) Project +(70) Project Output [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Input [5]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(66) HashAggregate +(71) HashAggregate Input [3]: [ps_partkey#X, ps_availqty#X, ps_supplycost#X] Keys [1]: [ps_partkey#X] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [ps_partkey#X, sum#X, isEmpty#X] -(67) Exchange +(72) Exchange Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) HashAggregate +(73) HashAggregate Input [3]: [ps_partkey#X, sum#X, isEmpty#X] Keys [1]: [ps_partkey#X] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [2]: [ps_partkey#X, sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X AS value#X] -(69) Filter +(74) Filter Input [2]: [ps_partkey#X, value#X] Condition : (isnotnull(value#X) AND (cast(value#X as decimal(38,6)) > Subquery subquery#X, [id=#X])) -(70) Exchange +(75) Exchange Input [2]: [ps_partkey#X, value#X] Arguments: rangepartitioning(value#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Sort +(76) Sort Input [2]: [ps_partkey#X, value#X] Arguments: [value#X DESC NULLS LAST], true, 0 -(72) AdaptiveSparkPlan +(77) AdaptiveSparkPlan Output [2]: [ps_partkey#X, value#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 42 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (120) +Subquery:1 Hosting operator id = 46 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (102) - +- ^ ProjectExecTransformer (100) - +- ^ RegularHashAggregateExecTransformer (99) - +- ^ RegularHashAggregateExecTransformer (98) - +- ^ ProjectExecTransformer (97) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (96) - :- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (85) - : :- ^ InputIteratorTransformer (80) - : : +- ShuffleQueryStage (78), Statistics(X) - : : +- ColumnarExchange (77) - : : +- ^ ProjectExecTransformer (75) - : : +- ^ FilterExecTransformer (74) - : : +- ^ Scan parquet (73) - : +- ^ InputIteratorTransformer (84) - : +- ShuffleQueryStage (82), Statistics(X) - : +- ReusedExchange (81) - +- ^ InputIteratorTransformer (95) - +- ShuffleQueryStage (93), Statistics(X) - +- ReusedExchange (92) + VeloxColumnarToRowExec (109) + +- ^ ProjectExecTransformer (107) + +- ^ RegularHashAggregateExecTransformer (106) + +- ^ RegularHashAggregateExecTransformer (105) + +- ^ ProjectExecTransformer (104) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (103) + :- ^ InputIteratorTransformer (98) + : +- ShuffleQueryStage (96), Statistics(X) + : +- ColumnarExchange (95) + : +- VeloxAppendBatches (94) + : +- ^ ProjectExecTransformer (92) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (91) + : :- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (90) + : +- ShuffleQueryStage (88), Statistics(X) + : +- ReusedExchange (87) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ReusedExchange (99) +- == Initial Plan == - HashAggregate (119) - +- HashAggregate (118) - +- Project (117) - +- ShuffledHashJoin Inner BuildRight (116) - :- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildRight (109) - : :- Exchange (105) - : : +- Filter (104) - : : +- Scan parquet (103) - : +- Exchange (108) - : +- Filter (107) - : +- Scan parquet (106) - +- Exchange (115) - +- Project (114) - +- Filter (113) - +- Scan parquet (112) - - -(73) Scan parquet + HashAggregate (126) + +- HashAggregate (125) + +- Project (124) + +- ShuffledHashJoin Inner BuildRight (123) + :- Exchange (118) + : +- Project (117) + : +- ShuffledHashJoin Inner BuildRight (116) + : :- Exchange (112) + : : +- Filter (111) + : : +- Scan parquet (110) + : +- Exchange (115) + : +- Filter (114) + : +- Scan parquet (113) + +- Exchange (122) + +- Project (121) + +- Filter (120) + +- Scan parquet (119) + + +(78) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(74) FilterExecTransformer +(79) FilterExecTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: isnotnull(ps_suppkey#X) -(75) ProjectExecTransformer +(80) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(76) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: false -(77) ColumnarExchange +(82) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] +Arguments: X + +(83) ColumnarExchange Input [4]: [hash_partition_key#X, ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X, ps_availqty#X, ps_supplycost#X], [plan_id=X], [id=#X] -(78) ShuffleQueryStage +(84) ShuffleQueryStage Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: X -(79) InputAdapter +(85) InputAdapter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(80) InputIteratorTransformer +(86) InputIteratorTransformer Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] -(81) ReusedExchange [Reuses operator id: 13] +(87) ReusedExchange [Reuses operator id: 15] Output [2]: [s_suppkey#X, s_nationkey#X] -(82) ShuffleQueryStage +(88) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(83) InputAdapter +(89) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(84) InputIteratorTransformer +(90) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(85) ShuffledHashJoinExecTransformer +(91) ShuffledHashJoinExecTransformer Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(86) ProjectExecTransformer +(92) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(87) WholeStageCodegenTransformer (X) +(93) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: false -(88) ColumnarExchange +(94) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] +Arguments: X + +(95) ColumnarExchange Input [4]: [hash_partition_key#X, ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [ps_availqty#X, ps_supplycost#X, s_nationkey#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(96) ShuffleQueryStage Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: X -(90) InputAdapter +(97) InputAdapter Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(91) InputIteratorTransformer +(98) InputIteratorTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] -(92) ReusedExchange [Reuses operator id: 28] +(99) ReusedExchange [Reuses operator id: 32] Output [1]: [n_nationkey#X] -(93) ShuffleQueryStage +(100) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(94) InputAdapter +(101) InputAdapter Input [1]: [n_nationkey#X] -(95) InputIteratorTransformer +(102) InputIteratorTransformer Input [1]: [n_nationkey#X] -(96) ShuffledHashJoinExecTransformer +(103) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(97) ProjectExecTransformer +(104) ProjectExecTransformer Output [3]: [ps_availqty#X, ps_supplycost#X, (ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))) AS _pre_X#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(98) RegularHashAggregateExecTransformer +(105) RegularHashAggregateExecTransformer Input [3]: [ps_availqty#X, ps_supplycost#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(99) RegularHashAggregateExecTransformer +(106) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(100) ProjectExecTransformer +(107) ProjectExecTransformer Output [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Input [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] -(101) WholeStageCodegenTransformer (X) +(108) WholeStageCodegenTransformer (X) Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: false -(102) VeloxColumnarToRowExec +(109) VeloxColumnarToRowExec Input [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(103) Scan parquet +(110) Scan parquet Output [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey)] ReadSchema: struct -(104) Filter +(111) Filter Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Condition : isnotnull(ps_suppkey#X) -(105) Exchange +(112) Exchange Input [3]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(113) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(107) Filter +(114) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(108) Exchange +(115) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(116) ShuffledHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(110) Project +(117) Project Output [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Input [5]: [ps_suppkey#X, ps_availqty#X, ps_supplycost#X, s_suppkey#X, s_nationkey#X] -(111) Exchange +(118) Exchange Input [3]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(119) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,GERMANY), IsNotNull(n_nationkey)] ReadSchema: struct -(113) Filter +(120) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = GERMANY)) AND isnotnull(n_nationkey#X)) -(114) Project +(121) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(115) Exchange +(122) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) ShuffledHashJoin +(123) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(117) Project +(124) Project Output [2]: [ps_availqty#X, ps_supplycost#X] Input [4]: [ps_availqty#X, ps_supplycost#X, s_nationkey#X, n_nationkey#X] -(118) HashAggregate +(125) HashAggregate Input [2]: [ps_availqty#X, ps_supplycost#X] Keys: [] Functions [1]: [partial_sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(119) HashAggregate +(126) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))] Aggregate Attributes [1]: [sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X] Results [1]: [(sum((ps_supplycost#X * cast(ps_availqty#X as decimal(10,0))))#X * 0.0001000000) AS (sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] -(120) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [1]: [(sum((ps_supplycost * ps_availqty)) * 0.0001000000)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt index 27765da815dd8..75948ca7ad8b8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/12.txt @@ -1,46 +1,49 @@ == Physical Plan == -AdaptiveSparkPlan (49) +AdaptiveSparkPlan (52) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (48) - +- Exchange (47) - +- HashAggregate (46) - +- Exchange (45) - +- HashAggregate (44) - +- Project (43) - +- ShuffledHashJoin Inner BuildLeft (42) - :- Exchange (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (41) - +- Project (40) - +- Filter (39) - +- Scan parquet (38) + Sort (51) + +- Exchange (50) + +- HashAggregate (49) + +- Exchange (48) + +- HashAggregate (47) + +- Project (46) + +- ShuffledHashJoin Inner BuildLeft (45) + :- Exchange (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (44) + +- Project (43) + +- Filter (42) + +- Scan parquet (41) (1) Scan parquet @@ -62,198 +65,210 @@ Input [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Arguments: ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_shipmode#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_shipmode#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_shipmode#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_shipmode#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [o_orderpriority#X, l_shipmode#X, CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X, CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END AS _pre_X#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [o_orderpriority#X, l_shipmode#X, _pre_X#X, _pre_X#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [4]: [hash(l_shipmode#X, 42) AS hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Input [3]: [l_shipmode#X, sum#X, sum#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [4]: [hash_partition_key#X, l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [l_shipmode#X, sum#X, sum#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [3]: [l_shipmode#X, sum#X, sum#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [3]: [l_shipmode#X, sum#X, sum#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] +Arguments: X + +(31) ColumnarExchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] -(35) Scan parquet +(38) Scan parquet Output [2]: [o_orderkey#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(36) Filter +(39) Filter Input [2]: [o_orderkey#X, o_orderpriority#X] Condition : isnotnull(o_orderkey#X) -(37) Exchange +(40) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(38) Scan parquet +(41) Scan parquet Output [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate), IsNotNull(l_shipdate), In(l_shipmode, [MAIL,SHIP]), GreaterThanOrEqual(l_receiptdate,1994-01-01), LessThan(l_receiptdate,1995-01-01), IsNotNull(l_orderkey)] ReadSchema: struct -(39) Filter +(42) Filter Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] Condition : ((((((((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND isnotnull(l_shipdate#X)) AND l_shipmode#X IN (MAIL,SHIP)) AND (l_commitdate#X < l_receiptdate#X)) AND (l_shipdate#X < l_commitdate#X)) AND (l_receiptdate#X >= 1994-01-01)) AND (l_receiptdate#X < 1995-01-01)) AND isnotnull(l_orderkey#X)) -(40) Project +(43) Project Output [2]: [l_orderkey#X, l_shipmode#X] Input [5]: [l_orderkey#X, l_shipdate#X, l_commitdate#X, l_receiptdate#X, l_shipmode#X] -(41) Exchange +(44) Exchange Input [2]: [l_orderkey#X, l_shipmode#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(42) ShuffledHashJoin +(45) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(43) Project +(46) Project Output [2]: [o_orderpriority#X, l_shipmode#X] Input [4]: [o_orderkey#X, o_orderpriority#X, l_orderkey#X, l_shipmode#X] -(44) HashAggregate +(47) HashAggregate Input [2]: [o_orderpriority#X, l_shipmode#X] Keys [1]: [l_shipmode#X] Functions [2]: [partial_sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), partial_sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum#X, sum#X] Results [3]: [l_shipmode#X, sum#X, sum#X] -(45) Exchange +(48) Exchange Input [3]: [l_shipmode#X, sum#X, sum#X] Arguments: hashpartitioning(l_shipmode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(46) HashAggregate +(49) HashAggregate Input [3]: [l_shipmode#X, sum#X, sum#X] Keys [1]: [l_shipmode#X] Functions [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END), sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)] Aggregate Attributes [2]: [sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X] Results [3]: [l_shipmode#X, sum(CASE WHEN ((o_orderpriority#X = 1-URGENT) OR (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS high_line_count#X, sum(CASE WHEN (NOT (o_orderpriority#X = 1-URGENT) AND NOT (o_orderpriority#X = 2-HIGH)) THEN 1 ELSE 0 END)#X AS low_line_count#X] -(47) Exchange +(50) Exchange Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: rangepartitioning(l_shipmode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(48) Sort +(51) Sort Input [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: [l_shipmode#X ASC NULLS FIRST], true, 0 -(49) AdaptiveSparkPlan +(52) AdaptiveSparkPlan Output [3]: [l_shipmode#X, high_line_count#X, low_line_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt index 79b9dcd18bd11..5a8bd7bbc9552 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/13.txt @@ -1,49 +1,52 @@ == Physical Plan == -AdaptiveSparkPlan (52) +AdaptiveSparkPlan (55) +- == Final Plan == - VeloxColumnarToRowExec (36) - +- ^ SortExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ ProjectExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ RegularHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (16) - :- ^ InputIteratorTransformer (7) - : +- ShuffleQueryStage (5), Statistics(X) - : +- ColumnarExchange (4) - : +- ^ ProjectExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ FilterExecTransformer (9) - +- ^ Scan parquet (8) + VeloxColumnarToRowExec (39) + +- ^ SortExecTransformer (37) + +- ^ InputIteratorTransformer (36) + +- ShuffleQueryStage (34), Statistics(X) + +- ColumnarExchange (33) + +- VeloxAppendBatches (32) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ ProjectExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftOuter BuildLeft (18) + :- ^ InputIteratorTransformer (8) + : +- ShuffleQueryStage (6), Statistics(X) + : +- ColumnarExchange (5) + : +- VeloxAppendBatches (4) + : +- ^ ProjectExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ FilterExecTransformer (10) + +- ^ Scan parquet (9) +- == Initial Plan == - Sort (51) - +- Exchange (50) - +- HashAggregate (49) - +- Exchange (48) - +- HashAggregate (47) - +- HashAggregate (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftOuter BuildRight (43) - :- Exchange (38) - : +- Scan parquet (37) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (54) + +- Exchange (53) + +- HashAggregate (52) + +- Exchange (51) + +- HashAggregate (50) + +- HashAggregate (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftOuter BuildRight (46) + :- Exchange (41) + : +- Scan parquet (40) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -60,225 +63,237 @@ Input [1]: [c_custkey#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(4) ColumnarExchange +(4) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(5) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(5) ShuffleQueryStage +(6) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(6) InputAdapter +(7) InputAdapter Input [1]: [c_custkey#X] -(7) InputIteratorTransformer +(8) InputIteratorTransformer Input [1]: [c_custkey#X] -(8) Scan parquet +(9) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(9) FilterExecTransformer +(10) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Arguments: ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(18) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [2]: [c_custkey#X, count(o_orderkey#X)#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [1]: [count(o_orderkey#X)#X AS c_count#X] Input [2]: [c_custkey#X, count(o_orderkey#X)#X] -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [3]: [hash(c_count#X, 42) AS hash_partition_key#X, c_count#X, count#X] Input [2]: [c_count#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [3]: [hash_partition_key#X, c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [c_count#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [2]: [c_count#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [2]: [c_count#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [2]: [c_count#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(30) ColumnarExchange +(32) VeloxAppendBatches +Input [2]: [c_count#X, custdist#X] +Arguments: X + +(33) ColumnarExchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [c_count#X, custdist#X] Arguments: X -(32) InputAdapter +(35) InputAdapter Input [2]: [c_count#X, custdist#X] -(33) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [c_count#X, custdist#X] -(34) SortExecTransformer +(37) SortExecTransformer Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(35) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [2]: [c_count#X, custdist#X] Arguments: false -(36) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [2]: [c_count#X, custdist#X] -(37) Scan parquet +(40) Scan parquet Output [1]: [c_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_comment), IsNotNull(o_custkey)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] Condition : ((isnotnull(o_comment#X) AND NOT o_comment#X LIKE %special%requests%) AND isnotnull(o_custkey#X)) -(41) Project +(44) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_comment#X] -(42) Exchange +(45) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftOuter Join condition: None -(44) Project +(47) Project Output [2]: [c_custkey#X, o_orderkey#X] Input [3]: [c_custkey#X, o_orderkey#X, o_custkey#X] -(45) HashAggregate +(48) HashAggregate Input [2]: [c_custkey#X, o_orderkey#X] Keys [1]: [c_custkey#X] Functions [1]: [partial_count(o_orderkey#X)] Aggregate Attributes [1]: [count#X] Results [2]: [c_custkey#X, count#X] -(46) HashAggregate +(49) HashAggregate Input [2]: [c_custkey#X, count#X] Keys [1]: [c_custkey#X] Functions [1]: [count(o_orderkey#X)] Aggregate Attributes [1]: [count(o_orderkey#X)#X] Results [1]: [count(o_orderkey#X)#X AS c_count#X] -(47) HashAggregate +(50) HashAggregate Input [1]: [c_count#X] Keys [1]: [c_count#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [c_count#X, count#X] -(48) Exchange +(51) Exchange Input [2]: [c_count#X, count#X] Arguments: hashpartitioning(c_count#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) HashAggregate +(52) HashAggregate Input [2]: [c_count#X, count#X] Keys [1]: [c_count#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [c_count#X, count(1)#X AS custdist#X] -(50) Exchange +(53) Exchange Input [2]: [c_count#X, custdist#X] Arguments: rangepartitioning(custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Sort +(54) Sort Input [2]: [c_count#X, custdist#X] Arguments: [custdist#X DESC NULLS LAST, c_count#X DESC NULLS LAST], true, 0 -(52) AdaptiveSparkPlan +(55) AdaptiveSparkPlan Output [2]: [c_count#X, custdist#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt index 4db67eada5624..c6f425f008685 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/14.txt @@ -1,36 +1,38 @@ == Physical Plan == -AdaptiveSparkPlan (35) +AdaptiveSparkPlan (37) +- == Final Plan == - VeloxColumnarToRowExec (23) - +- ^ ProjectExecTransformer (21) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (25) + +- ^ ProjectExecTransformer (23) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (34) - +- HashAggregate (33) - +- Project (32) - +- ShuffledHashJoin Inner BuildRight (31) - :- Exchange (27) - : +- Project (26) - : +- Filter (25) - : +- Scan parquet (24) - +- Exchange (30) - +- Filter (29) - +- Scan parquet (28) + HashAggregate (36) + +- HashAggregate (35) + +- Project (34) + +- ShuffledHashJoin Inner BuildRight (33) + :- Exchange (29) + : +- Project (28) + : +- Filter (27) + : +- Scan parquet (26) + +- Exchange (32) + +- Filter (31) + +- Scan parquet (30) (1) Scan parquet @@ -52,146 +54,154 @@ Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [2]: [p_partkey#X, p_type#X] Arguments: isnotnull(p_partkey#X) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_type#X] Input [2]: [p_partkey#X, p_type#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_type#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [p_partkey#X, p_type#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [p_partkey#X, p_type#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [p_partkey#X, p_type#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [l_extendedprice#X, l_discount#X, p_type#X, CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [5]: [l_extendedprice#X, l_discount#X, p_type#X, _pre_X#X, _pre_X#X] Keys: [] Functions [2]: [partial_sum(_pre_X#X), partial_sum(_pre_X#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(21) ProjectExecTransformer +(23) ProjectExecTransformer Output [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] Input [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(22) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [1]: [promo_revenue#X] Arguments: false -(23) VeloxColumnarToRowExec +(25) VeloxColumnarToRowExec Input [1]: [promo_revenue#X] -(24) Scan parquet +(26) Scan parquet Output [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-09-01), LessThan(l_shipdate,1995-10-01), IsNotNull(l_partkey)] ReadSchema: struct -(25) Filter +(27) Filter Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-09-01)) AND (l_shipdate#X < 1995-10-01)) AND isnotnull(l_partkey#X)) -(26) Project +(28) Project Output [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_partkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) Exchange +(29) Exchange Input [3]: [l_partkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(28) Scan parquet +(30) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_partkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [2]: [p_partkey#X, p_type#X] Condition : isnotnull(p_partkey#X) -(30) Exchange +(32) Exchange Input [2]: [p_partkey#X, p_type#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) ShuffledHashJoin +(33) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(32) Project +(34) Project Output [3]: [l_extendedprice#X, l_discount#X, p_type#X] Input [5]: [l_partkey#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_type#X] -(33) HashAggregate +(35) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, p_type#X] Keys: [] Functions [2]: [partial_sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] -(34) HashAggregate +(36) HashAggregate Input [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Keys: [] Functions [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END), sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [((100.00 * sum(CASE WHEN StartsWith(p_type#X, PROMO) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END)#X) / sum((l_extendedprice#X * (1 - l_discount#X)))#X) AS promo_revenue#X] -(35) AdaptiveSparkPlan +(37) AdaptiveSparkPlan Output [1]: [promo_revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt index c8f4e2c84ac3e..5592b9e73add7 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/15.txt @@ -1,43 +1,45 @@ == Physical Plan == -AdaptiveSparkPlan (42) +AdaptiveSparkPlan (44) +- == Final Plan == - VeloxColumnarToRowExec (27) - +- AQEShuffleRead (26) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (21) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ FilterExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ InputIteratorTransformer (18) - +- ShuffleQueryStage (16), Statistics(X) - +- ColumnarExchange (15) - +- ^ ProjectExecTransformer (13) - +- ^ FlushableHashAggregateExecTransformer (12) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (29) + +- AQEShuffleRead (28) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- VeloxAppendBatches (25) + +- ^ ProjectExecTransformer (23) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (22) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ FilterExecTransformer (21) + +- ^ RegularHashAggregateExecTransformer (20) + +- ^ InputIteratorTransformer (19) + +- ShuffleQueryStage (17), Statistics(X) + +- ColumnarExchange (16) + +- ^ ProjectExecTransformer (14) + +- ^ FlushableHashAggregateExecTransformer (13) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (41) - +- Exchange (40) - +- Project (39) - +- ShuffledHashJoin Inner BuildLeft (38) - :- Exchange (30) - : +- Filter (29) - : +- Scan parquet (28) - +- Filter (37) - +- HashAggregate (36) - +- Exchange (35) - +- HashAggregate (34) - +- Project (33) - +- Filter (32) - +- Scan parquet (31) + Sort (43) + +- Exchange (42) + +- Project (41) + +- ShuffledHashJoin Inner BuildLeft (40) + :- Exchange (32) + : +- Filter (31) + : +- Scan parquet (30) + +- Filter (39) + +- HashAggregate (38) + +- Exchange (37) + +- HashAggregate (36) + +- Project (35) + +- Filter (34) + +- Scan parquet (33) (1) Scan parquet @@ -59,330 +61,338 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_phone#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) FlushableHashAggregateExecTransformer +(13) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(13) ProjectExecTransformer +(14) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(14) WholeStageCodegenTransformer (X) +(15) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(15) ColumnarExchange +(16) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(16) ShuffleQueryStage +(17) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(17) InputAdapter +(18) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(18) InputIteratorTransformer +(19) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(19) RegularHashAggregateExecTransformer +(20) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(20) FilterExecTransformer +(21) FilterExecTransformer Input [2]: [supplier_no#X, total_revenue#X] Arguments: (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(21) ShuffledHashJoinExecTransformer +(22) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(22) ProjectExecTransformer +(23) ProjectExecTransformer Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(23) WholeStageCodegenTransformer (X) +(24) WholeStageCodegenTransformer (X) Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: false -(24) ColumnarExchange +(25) VeloxAppendBatches +Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] +Arguments: X + +(26) ColumnarExchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: X -(26) AQEShuffleRead +(28) AQEShuffleRead Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: local -(27) VeloxColumnarToRowExec +(29) VeloxColumnarToRowExec Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] -(28) Scan parquet +(30) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey)] ReadSchema: struct -(29) Filter +(31) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Condition : isnotnull(s_suppkey#X) -(30) Exchange +(32) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(31) Scan parquet +(33) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01), IsNotNull(l_suppkey)] ReadSchema: struct -(32) Filter +(34) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) AND isnotnull(l_suppkey#X)) -(33) Project +(35) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(34) HashAggregate +(36) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(35) Exchange +(37) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(36) HashAggregate +(38) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X AS supplier_no#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(37) Filter +(39) Filter Input [2]: [supplier_no#X, total_revenue#X] Condition : (isnotnull(total_revenue#X) AND (total_revenue#X = Subquery subquery#X, [id=#X])) -(38) ShuffledHashJoin +(40) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [supplier_no#X] Join type: Inner Join condition: None -(39) Project +(41) Project Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Input [6]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, supplier_no#X, total_revenue#X] -(40) Exchange +(42) Exchange Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: rangepartitioning(s_suppkey#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Sort +(43) Sort Input [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: [s_suppkey#X ASC NULLS FIRST], true, 0 -(42) AdaptiveSparkPlan +(44) AdaptiveSparkPlan Output [5]: [s_suppkey#X, s_name#X, s_address#X, s_phone#X, total_revenue#X] Arguments: isFinalPlan=true ===== Subqueries ===== -Subquery:1 Hosting operator id = 20 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (67) +Subquery:1 Hosting operator id = 21 Hosting Expression = Subquery subquery#X, [id=#X] +AdaptiveSparkPlan (69) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ RegularHashAggregateExecTransformer (55) - +- ^ ProjectExecTransformer (54) - +- ^ RegularHashAggregateExecTransformer (53) - +- ^ InputIteratorTransformer (52) - +- ShuffleQueryStage (50), Statistics(X) - +- ColumnarExchange (49) - +- ^ ProjectExecTransformer (47) - +- ^ FlushableHashAggregateExecTransformer (46) - +- ^ ProjectExecTransformer (45) - +- ^ FilterExecTransformer (44) - +- ^ Scan parquet (43) + VeloxColumnarToRowExec (60) + +- ^ RegularHashAggregateExecTransformer (58) + +- ^ RegularHashAggregateExecTransformer (57) + +- ^ ProjectExecTransformer (56) + +- ^ RegularHashAggregateExecTransformer (55) + +- ^ InputIteratorTransformer (54) + +- ShuffleQueryStage (52), Statistics(X) + +- ColumnarExchange (51) + +- ^ ProjectExecTransformer (49) + +- ^ FlushableHashAggregateExecTransformer (48) + +- ^ ProjectExecTransformer (47) + +- ^ FilterExecTransformer (46) + +- ^ Scan parquet (45) +- == Initial Plan == - HashAggregate (66) - +- HashAggregate (65) - +- HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (68) + +- HashAggregate (67) + +- HashAggregate (66) + +- Exchange (65) + +- HashAggregate (64) + +- Project (63) + +- Filter (62) + +- Scan parquet (61) -(43) Scan parquet +(45) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(44) FilterExecTransformer +(46) FilterExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(45) ProjectExecTransformer +(47) ProjectExecTransformer Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(46) FlushableHashAggregateExecTransformer +(48) FlushableHashAggregateExecTransformer Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(47) ProjectExecTransformer +(49) ProjectExecTransformer Output [4]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(48) WholeStageCodegenTransformer (X) +(50) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: false -(49) ColumnarExchange +(51) ColumnarExchange Input [4]: [hash_partition_key#X, l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_suppkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(50) ShuffleQueryStage +(52) ShuffleQueryStage Output [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: X -(51) InputAdapter +(53) InputAdapter Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(52) InputIteratorTransformer +(54) InputIteratorTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] -(53) RegularHashAggregateExecTransformer +(55) RegularHashAggregateExecTransformer Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(54) ProjectExecTransformer +(56) ProjectExecTransformer Output [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] Input [2]: [l_suppkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(55) RegularHashAggregateExecTransformer +(57) RegularHashAggregateExecTransformer Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(56) RegularHashAggregateExecTransformer +(58) RegularHashAggregateExecTransformer Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(57) WholeStageCodegenTransformer (X) +(59) WholeStageCodegenTransformer (X) Input [1]: [max(total_revenue)#X] Arguments: false -(58) VeloxColumnarToRowExec +(60) VeloxColumnarToRowExec Input [1]: [max(total_revenue)#X] -(59) Scan parquet +(61) Scan parquet Output [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1996-01-01), LessThan(l_shipdate,1996-04-01)] ReadSchema: struct -(60) Filter +(62) Filter Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1996-01-01)) AND (l_shipdate#X < 1996-04-01)) -(61) Project +(63) Project Output [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(62) HashAggregate +(64) HashAggregate Input [3]: [l_suppkey#X, l_extendedprice#X, l_discount#X] Keys [1]: [l_suppkey#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_suppkey#X, sum#X, isEmpty#X] -(63) Exchange +(65) Exchange Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(66) HashAggregate Input [3]: [l_suppkey#X, sum#X, isEmpty#X] Keys [1]: [l_suppkey#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS total_revenue#X] -(65) HashAggregate +(67) HashAggregate Input [1]: [total_revenue#X] Keys: [] Functions [1]: [partial_max(total_revenue#X)] Aggregate Attributes [1]: [max#X] Results [1]: [max#X] -(66) HashAggregate +(68) HashAggregate Input [1]: [max#X] Keys: [] Functions [1]: [max(total_revenue#X)] Aggregate Attributes [1]: [max(total_revenue#X)#X] Results [1]: [max(total_revenue#X)#X AS max(total_revenue)#X] -(67) AdaptiveSparkPlan +(69) AdaptiveSparkPlan Output [1]: [max(total_revenue)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt index 247b853fb1c4a..73703d5baf678 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/16.txt @@ -1,59 +1,62 @@ == Physical Plan == -AdaptiveSparkPlan (64) +AdaptiveSparkPlan (67) +- == Final Plan == - VeloxColumnarToRowExec (42) - +- ^ SortExecTransformer (40) - +- ^ InputIteratorTransformer (39) - +- ShuffleQueryStage (37), Statistics(X) - +- ColumnarExchange (36) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ InputIteratorTransformer (33) - +- ShuffleQueryStage (31), Statistics(X) - +- ColumnarExchange (30) - +- ^ ProjectExecTransformer (28) - +- ^ FlushableHashAggregateExecTransformer (27) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (45) + +- ^ SortExecTransformer (43) + +- ^ InputIteratorTransformer (42) + +- ShuffleQueryStage (40), Statistics(X) + +- ColumnarExchange (39) + +- VeloxAppendBatches (38) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- ^ ProjectExecTransformer (30) + +- ^ FlushableHashAggregateExecTransformer (29) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (63) - +- Exchange (62) - +- HashAggregate (61) - +- Exchange (60) - +- HashAggregate (59) - +- HashAggregate (58) - +- Exchange (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (50) - : +- BroadcastHashJoin LeftAnti BuildRight (49) - : :- Filter (44) - : : +- Scan parquet (43) - : +- BroadcastExchange (48) - : +- Project (47) - : +- Filter (46) - : +- Scan parquet (45) - +- Exchange (53) - +- Filter (52) - +- Scan parquet (51) + Sort (66) + +- Exchange (65) + +- HashAggregate (64) + +- Exchange (63) + +- HashAggregate (62) + +- HashAggregate (61) + +- Exchange (60) + +- HashAggregate (59) + +- Project (58) + +- ShuffledHashJoin Inner BuildRight (57) + :- Exchange (53) + : +- BroadcastHashJoin LeftAnti BuildRight (52) + : :- Filter (47) + : : +- Scan parquet (46) + : +- BroadcastExchange (51) + : +- Project (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -75,273 +78,285 @@ Input [2]: [ps_partkey#X, ps_suppkey#X] Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [ps_partkey#X, ps_suppkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [ps_partkey#X, ps_suppkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [ps_partkey#X, ps_suppkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_type#X, p_size#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, ps_suppkey#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(27) FlushableHashAggregateExecTransformer +(29) FlushableHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(28) ProjectExecTransformer +(30) ProjectExecTransformer Output [5]: [hash(p_brand#X, p_type#X, p_size#X, 42) AS hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(29) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: false -(30) ColumnarExchange +(32) ColumnarExchange Input [5]: [hash_partition_key#X, p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [p_brand#X, p_type#X, p_size#X, count#X], [plan_id=X], [id=#X] -(31) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: X -(32) InputAdapter +(34) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(33) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(35) WholeStageCodegenTransformer (X) +(37) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(36) ColumnarExchange +(38) VeloxAppendBatches +Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] +Arguments: X + +(39) ColumnarExchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(40) ShuffleQueryStage Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: X -(38) InputAdapter +(41) InputAdapter Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(39) InputIteratorTransformer +(42) InputIteratorTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(40) SortExecTransformer +(43) SortExecTransformer Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(41) WholeStageCodegenTransformer (X) +(44) WholeStageCodegenTransformer (X) Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: false -(42) VeloxColumnarToRowExec +(45) VeloxColumnarToRowExec Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] -(43) Scan parquet +(46) Scan parquet Output [2]: [ps_partkey#X, ps_suppkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_partkey)] ReadSchema: struct -(44) Filter +(47) Filter Input [2]: [ps_partkey#X, ps_suppkey#X] Condition : isnotnull(ps_partkey#X) -(45) Scan parquet +(48) Scan parquet Output [2]: [s_suppkey#X, s_comment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_comment)] ReadSchema: struct -(46) Filter +(49) Filter Input [2]: [s_suppkey#X, s_comment#X] Condition : (isnotnull(s_comment#X) AND s_comment#X LIKE %Customer%Complaints%) -(47) Project +(50) Project Output [1]: [s_suppkey#X] Input [2]: [s_suppkey#X, s_comment#X] -(48) BroadcastExchange +(51) BroadcastExchange Input [1]: [s_suppkey#X] Arguments: HashedRelationBroadcastMode(List(input[0, bigint, true]),true), [plan_id=X] -(49) BroadcastHashJoin +(52) BroadcastHashJoin Left keys [1]: [ps_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: LeftAnti Join condition: None -(50) Exchange +(53) Exchange Input [2]: [ps_partkey#X, ps_suppkey#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) Scan parquet +(54) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#X)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct -(52) Filter +(55) Filter Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Condition : (((((isnotnull(p_brand#X) AND isnotnull(p_type#X)) AND NOT (p_brand#X = Brand#X)) AND NOT StartsWith(p_type#X, MEDIUM POLISHED)) AND p_size#X IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#X)) -(53) Exchange +(56) Exchange Input [4]: [p_partkey#X, p_brand#X, p_type#X, p_size#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(57) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(55) Project +(58) Project Output [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Input [6]: [ps_partkey#X, ps_suppkey#X, p_partkey#X, p_brand#X, p_type#X, p_size#X] -(56) HashAggregate +(59) HashAggregate Input [4]: [ps_suppkey#X, p_brand#X, p_type#X, p_size#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(57) Exchange +(60) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(58) HashAggregate +(61) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Functions: [] Aggregate Attributes: [] Results [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] -(59) HashAggregate +(62) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, ps_suppkey#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [partial_count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count#X] -(60) Exchange +(63) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Arguments: hashpartitioning(p_brand#X, p_type#X, p_size#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(61) HashAggregate +(64) HashAggregate Input [4]: [p_brand#X, p_type#X, p_size#X, count#X] Keys [3]: [p_brand#X, p_type#X, p_size#X] Functions [1]: [count(distinct ps_suppkey#X)] Aggregate Attributes [1]: [count(ps_suppkey#X)#X] Results [4]: [p_brand#X, p_type#X, p_size#X, count(ps_suppkey#X)#X AS supplier_cnt#X] -(62) Exchange +(65) Exchange Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: rangepartitioning(supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(63) Sort +(66) Sort Input [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: [supplier_cnt#X DESC NULLS LAST, p_brand#X ASC NULLS FIRST, p_type#X ASC NULLS FIRST, p_size#X ASC NULLS FIRST], true, 0 -(64) AdaptiveSparkPlan +(67) AdaptiveSparkPlan Output [4]: [p_brand#X, p_type#X, p_size#X, supplier_cnt#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt index fad48184fed17..081d1e9720145 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/17.txt @@ -1,56 +1,58 @@ == Physical Plan == -AdaptiveSparkPlan (57) +AdaptiveSparkPlan (59) +- == Final Plan == - VeloxColumnarToRowExec (37) - +- ^ ProjectExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ RegularHashAggregateExecTransformer (33) - +- ^ ProjectExecTransformer (32) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (31) - :- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ FilterExecTransformer (30) - +- ^ ProjectExecTransformer (29) - +- ^ RegularHashAggregateExecTransformer (28) - +- ^ InputIteratorTransformer (27) - +- ShuffleQueryStage (25), Statistics(X) - +- ColumnarExchange (24) - +- ^ ProjectExecTransformer (22) - +- ^ FlushableHashAggregateExecTransformer (21) - +- ^ FilterExecTransformer (20) - +- ^ Scan parquet (19) + VeloxColumnarToRowExec (39) + +- ^ ProjectExecTransformer (37) + +- ^ RegularHashAggregateExecTransformer (36) + +- ^ RegularHashAggregateExecTransformer (35) + +- ^ ProjectExecTransformer (34) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (33) + :- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ FilterExecTransformer (32) + +- ^ ProjectExecTransformer (31) + +- ^ RegularHashAggregateExecTransformer (30) + +- ^ InputIteratorTransformer (29) + +- ShuffleQueryStage (27), Statistics(X) + +- ColumnarExchange (26) + +- ^ ProjectExecTransformer (24) + +- ^ FlushableHashAggregateExecTransformer (23) + +- ^ FilterExecTransformer (22) + +- ^ Scan parquet (21) +- == Initial Plan == - HashAggregate (56) - +- HashAggregate (55) - +- Project (54) - +- ShuffledHashJoin Inner BuildRight (53) - :- Project (46) - : +- ShuffledHashJoin Inner BuildRight (45) - : :- Exchange (40) - : : +- Filter (39) - : : +- Scan parquet (38) - : +- Exchange (44) - : +- Project (43) - : +- Filter (42) - : +- Scan parquet (41) - +- Filter (52) - +- HashAggregate (51) - +- Exchange (50) - +- HashAggregate (49) - +- Filter (48) - +- Scan parquet (47) + HashAggregate (58) + +- HashAggregate (57) + +- Project (56) + +- ShuffledHashJoin Inner BuildRight (55) + :- Project (48) + : +- ShuffledHashJoin Inner BuildRight (47) + : :- Exchange (42) + : : +- Filter (41) + : : +- Scan parquet (40) + : +- Exchange (46) + : +- Project (45) + : +- Filter (44) + : +- Scan parquet (43) + +- Filter (54) + +- HashAggregate (53) + +- Exchange (52) + +- HashAggregate (51) + +- Filter (50) + +- Scan parquet (49) (1) Scan parquet @@ -72,254 +74,262 @@ Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [p_partkey#X, p_brand#X, p_container#X] Arguments: ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [p_partkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [p_partkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(19) Scan parquet +(21) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(20) FilterExecTransformer +(22) FilterExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Arguments: isnotnull(l_partkey#X) -(21) FlushableHashAggregateExecTransformer +(23) FlushableHashAggregateExecTransformer Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(22) ProjectExecTransformer +(24) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, sum#X, count#X] Input [3]: [l_partkey#X, sum#X, count#X] -(23) WholeStageCodegenTransformer (X) +(25) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: false -(24) ColumnarExchange +(26) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, sum#X, count#X], [plan_id=X], [id=#X] -(25) ShuffleQueryStage +(27) ShuffleQueryStage Output [3]: [l_partkey#X, sum#X, count#X] Arguments: X -(26) InputAdapter +(28) InputAdapter Input [3]: [l_partkey#X, sum#X, count#X] -(27) InputIteratorTransformer +(29) InputIteratorTransformer Input [3]: [l_partkey#X, sum#X, count#X] -(28) RegularHashAggregateExecTransformer +(30) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [l_partkey#X, avg(l_quantity#X)#X] -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] Input [2]: [l_partkey#X, avg(l_quantity#X)#X] -(30) FilterExecTransformer +(32) FilterExecTransformer Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Arguments: isnotnull((0.2 * avg(l_quantity))#X) -(31) ShuffledHashJoinExecTransformer +(33) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(32) ProjectExecTransformer +(34) ProjectExecTransformer Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(33) RegularHashAggregateExecTransformer +(35) RegularHashAggregateExecTransformer Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(34) RegularHashAggregateExecTransformer +(36) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [sum(l_extendedprice#X)#X] -(35) ProjectExecTransformer +(37) ProjectExecTransformer Output [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] Input [1]: [sum(l_extendedprice#X)#X] -(36) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [1]: [avg_yearly#X] Arguments: false -(37) VeloxColumnarToRowExec +(39) VeloxColumnarToRowExec Input [1]: [avg_yearly#X] -(38) Scan parquet +(40) Scan parquet Output [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_quantity)] ReadSchema: struct -(39) Filter +(41) Filter Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Condition : (isnotnull(l_partkey#X) AND isnotnull(l_quantity#X)) -(40) Exchange +(42) Exchange Input [3]: [l_partkey#X, l_quantity#X, l_extendedprice#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(41) Scan parquet +(43) Scan parquet Output [3]: [p_partkey#X, p_brand#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#X), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct -(42) Filter +(44) Filter Input [3]: [p_partkey#X, p_brand#X, p_container#X] Condition : ((((isnotnull(p_brand#X) AND isnotnull(p_container#X)) AND (p_brand#X = Brand#X)) AND (p_container#X = MED BOX)) AND isnotnull(p_partkey#X)) -(43) Project +(45) Project Output [1]: [p_partkey#X] Input [3]: [p_partkey#X, p_brand#X, p_container#X] -(44) Exchange +(46) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) ShuffledHashJoin +(47) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: None -(46) Project +(48) Project Output [3]: [l_quantity#X, l_extendedprice#X, p_partkey#X] Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, p_partkey#X] -(47) Scan parquet +(49) Scan parquet Output [2]: [l_partkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct -(48) Filter +(50) Filter Input [2]: [l_partkey#X, l_quantity#X] Condition : isnotnull(l_partkey#X) -(49) HashAggregate +(51) HashAggregate Input [2]: [l_partkey#X, l_quantity#X] Keys [1]: [l_partkey#X] Functions [1]: [partial_avg(l_quantity#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [3]: [l_partkey#X, sum#X, count#X] -(50) Exchange +(52) Exchange Input [3]: [l_partkey#X, sum#X, count#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(51) HashAggregate +(53) HashAggregate Input [3]: [l_partkey#X, sum#X, count#X] Keys [1]: [l_partkey#X] Functions [1]: [avg(l_quantity#X)] Aggregate Attributes [1]: [avg(l_quantity#X)#X] Results [2]: [(0.2 * avg(l_quantity#X)#X) AS (0.2 * avg(l_quantity))#X, l_partkey#X] -(52) Filter +(54) Filter Input [2]: [(0.2 * avg(l_quantity))#X, l_partkey#X] Condition : isnotnull((0.2 * avg(l_quantity))#X) -(53) ShuffledHashJoin +(55) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: (cast(l_quantity#X as decimal(18,7)) < (0.2 * avg(l_quantity))#X) -(54) Project +(56) Project Output [1]: [l_extendedprice#X] Input [5]: [l_quantity#X, l_extendedprice#X, p_partkey#X, (0.2 * avg(l_quantity))#X, l_partkey#X] -(55) HashAggregate +(57) HashAggregate Input [1]: [l_extendedprice#X] Keys: [] Functions [1]: [partial_sum(l_extendedprice#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(56) HashAggregate +(58) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum(l_extendedprice#X)] Aggregate Attributes [1]: [sum(l_extendedprice#X)#X] Results [1]: [(sum(l_extendedprice#X)#X / 7.0) AS avg_yearly#X] -(57) AdaptiveSparkPlan +(59) AdaptiveSparkPlan Output [1]: [avg_yearly#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt index 5a6f040643490..8c55d747d0aea 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/18.txt @@ -1,89 +1,94 @@ == Physical Plan == -AdaptiveSparkPlan (97) +AdaptiveSparkPlan (102) +- == Final Plan == - VeloxColumnarToRowExec (64) - +- TakeOrderedAndProjectExecTransformer (63) - +- ^ RegularHashAggregateExecTransformer (61) - +- ^ RegularHashAggregateExecTransformer (60) - +- ^ ProjectExecTransformer (59) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (58) - :- ^ InputIteratorTransformer (41) - : +- ShuffleQueryStage (39), Statistics(X) - : +- ColumnarExchange (38) - : +- ^ ProjectExecTransformer (36) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (35) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (34) - : +- ShuffleQueryStage (32), Statistics(X) - : +- ColumnarExchange (31) - : +- ^ ProjectExecTransformer (29) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) - : :- ^ InputIteratorTransformer (16) - : : +- ShuffleQueryStage (14), Statistics(X) - : : +- ColumnarExchange (13) - : : +- ^ ProjectExecTransformer (11) - : : +- ^ FilterExecTransformer (10) - : : +- ^ Scan parquet (9) - : +- ^ ProjectExecTransformer (27) - : +- ^ FilterExecTransformer (26) - : +- ^ RegularHashAggregateExecTransformer (25) - : +- ^ InputIteratorTransformer (24) - : +- ShuffleQueryStage (22), Statistics(X) - : +- ColumnarExchange (21) - : +- ^ ProjectExecTransformer (19) - : +- ^ FlushableHashAggregateExecTransformer (18) - : +- ^ Scan parquet (17) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (57) - :- ^ InputIteratorTransformer (49) - : +- ShuffleQueryStage (47), Statistics(X) - : +- ColumnarExchange (46) - : +- ^ ProjectExecTransformer (44) - : +- ^ FilterExecTransformer (43) - : +- ^ Scan parquet (42) - +- ^ ProjectExecTransformer (56) - +- ^ FilterExecTransformer (55) - +- ^ RegularHashAggregateExecTransformer (54) - +- ^ InputIteratorTransformer (53) - +- ShuffleQueryStage (51), Statistics(X) - +- ReusedExchange (50) + VeloxColumnarToRowExec (69) + +- TakeOrderedAndProjectExecTransformer (68) + +- ^ RegularHashAggregateExecTransformer (66) + +- ^ RegularHashAggregateExecTransformer (65) + +- ^ ProjectExecTransformer (64) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (63) + :- ^ InputIteratorTransformer (45) + : +- ShuffleQueryStage (43), Statistics(X) + : +- ColumnarExchange (42) + : +- VeloxAppendBatches (41) + : +- ^ ProjectExecTransformer (39) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (38) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (37) + : +- ShuffleQueryStage (35), Statistics(X) + : +- ColumnarExchange (34) + : +- VeloxAppendBatches (33) + : +- ^ ProjectExecTransformer (31) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (30) + : :- ^ InputIteratorTransformer (18) + : : +- ShuffleQueryStage (16), Statistics(X) + : : +- ColumnarExchange (15) + : : +- VeloxAppendBatches (14) + : : +- ^ ProjectExecTransformer (12) + : : +- ^ FilterExecTransformer (11) + : : +- ^ Scan parquet (10) + : +- ^ ProjectExecTransformer (29) + : +- ^ FilterExecTransformer (28) + : +- ^ RegularHashAggregateExecTransformer (27) + : +- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- ^ ProjectExecTransformer (21) + : +- ^ FlushableHashAggregateExecTransformer (20) + : +- ^ Scan parquet (19) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) + :- ^ InputIteratorTransformer (54) + : +- ShuffleQueryStage (52), Statistics(X) + : +- ColumnarExchange (51) + : +- VeloxAppendBatches (50) + : +- ^ ProjectExecTransformer (48) + : +- ^ FilterExecTransformer (47) + : +- ^ Scan parquet (46) + +- ^ ProjectExecTransformer (61) + +- ^ FilterExecTransformer (60) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ReusedExchange (55) +- == Initial Plan == - TakeOrderedAndProject (96) - +- HashAggregate (95) - +- HashAggregate (94) - +- Project (93) - +- ShuffledHashJoin Inner BuildRight (92) - :- Exchange (81) - : +- Project (80) - : +- ShuffledHashJoin Inner BuildLeft (79) - : :- Exchange (67) - : : +- Filter (66) - : : +- Scan parquet (65) - : +- Exchange (78) - : +- ShuffledHashJoin LeftSemi BuildRight (77) - : :- Exchange (70) - : : +- Filter (69) - : : +- Scan parquet (68) - : +- Project (76) - : +- Filter (75) - : +- HashAggregate (74) - : +- Exchange (73) - : +- HashAggregate (72) - : +- Scan parquet (71) - +- ShuffledHashJoin LeftSemi BuildRight (91) - :- Exchange (84) - : +- Filter (83) - : +- Scan parquet (82) - +- Project (90) - +- Filter (89) - +- HashAggregate (88) - +- Exchange (87) - +- HashAggregate (86) - +- Scan parquet (85) + TakeOrderedAndProject (101) + +- HashAggregate (100) + +- HashAggregate (99) + +- Project (98) + +- ShuffledHashJoin Inner BuildRight (97) + :- Exchange (86) + : +- Project (85) + : +- ShuffledHashJoin Inner BuildLeft (84) + : :- Exchange (72) + : : +- Filter (71) + : : +- Scan parquet (70) + : +- Exchange (83) + : +- ShuffledHashJoin LeftSemi BuildRight (82) + : :- Exchange (75) + : : +- Filter (74) + : : +- Scan parquet (73) + : +- Project (81) + : +- Filter (80) + : +- HashAggregate (79) + : +- Exchange (78) + : +- HashAggregate (77) + : +- Scan parquet (76) + +- ShuffledHashJoin LeftSemi BuildRight (96) + :- Exchange (89) + : +- Filter (88) + : +- Scan parquet (87) + +- Project (95) + +- Filter (94) + +- HashAggregate (93) + +- Exchange (92) + +- HashAggregate (91) + +- Scan parquet (90) (1) Scan parquet @@ -105,428 +110,448 @@ Input [2]: [c_custkey#X, c_name#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_name#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_name#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_name#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(26) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(27) ProjectExecTransformer +(29) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(28) ShuffledHashJoinExecTransformer +(30) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(29) ProjectExecTransformer +(31) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(30) WholeStageCodegenTransformer (X) +(32) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(31) ColumnarExchange +(33) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(34) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(32) ShuffleQueryStage +(35) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(33) InputAdapter +(36) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(34) InputIteratorTransformer +(37) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(35) ShuffledHashJoinExecTransformer +(38) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(36) ProjectExecTransformer +(39) ProjectExecTransformer Output [6]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(37) WholeStageCodegenTransformer (X) +(40) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: false -(38) ColumnarExchange +(41) VeloxAppendBatches +Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] +Arguments: X + +(42) ColumnarExchange Input [6]: [hash_partition_key#X, c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X], [plan_id=X], [id=#X] -(39) ShuffleQueryStage +(43) ShuffleQueryStage Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: X -(40) InputAdapter +(44) InputAdapter Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(41) InputIteratorTransformer +(45) InputIteratorTransformer Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] -(42) Scan parquet +(46) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(43) FilterExecTransformer +(47) FilterExecTransformer Input [2]: [l_orderkey#X, l_quantity#X] Arguments: isnotnull(l_orderkey#X) -(44) ProjectExecTransformer +(48) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X] Input [2]: [l_orderkey#X, l_quantity#X] -(45) WholeStageCodegenTransformer (X) +(49) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: false -(46) ColumnarExchange +(50) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] +Arguments: X + +(51) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X], [plan_id=X], [id=#X] -(47) ShuffleQueryStage +(52) ShuffleQueryStage Output [2]: [l_orderkey#X, l_quantity#X] Arguments: X -(48) InputAdapter +(53) InputAdapter Input [2]: [l_orderkey#X, l_quantity#X] -(49) InputIteratorTransformer +(54) InputIteratorTransformer Input [2]: [l_orderkey#X, l_quantity#X] -(50) ReusedExchange [Reuses operator id: 21] +(55) ReusedExchange [Reuses operator id: 23] Output [3]: [l_orderkey#X, sum#X, isEmpty#X] -(51) ShuffleQueryStage +(56) ShuffleQueryStage Output [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: X -(52) InputAdapter +(57) InputAdapter Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(53) InputIteratorTransformer +(58) InputIteratorTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] -(54) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(55) FilterExecTransformer +(60) FilterExecTransformer Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Arguments: (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(56) ProjectExecTransformer +(61) ProjectExecTransformer Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(57) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(58) ShuffledHashJoinExecTransformer +(63) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(59) ProjectExecTransformer +(64) ProjectExecTransformer Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(60) RegularHashAggregateExecTransformer +(65) RegularHashAggregateExecTransformer Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(61) RegularHashAggregateExecTransformer +(66) RegularHashAggregateExecTransformer Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(62) WholeStageCodegenTransformer (X) +(67) WholeStageCodegenTransformer (X) Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: false -(63) TakeOrderedAndProjectExecTransformer +(68) TakeOrderedAndProjectExecTransformer Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X], 0 -(64) VeloxColumnarToRowExec +(69) VeloxColumnarToRowExec Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(65) Scan parquet +(70) Scan parquet Output [2]: [c_custkey#X, c_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey)] ReadSchema: struct -(66) Filter +(71) Filter Input [2]: [c_custkey#X, c_name#X] Condition : isnotnull(c_custkey#X) -(67) Exchange +(72) Exchange Input [2]: [c_custkey#X, c_name#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(68) Scan parquet +(73) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(69) Filter +(74) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Condition : (isnotnull(o_custkey#X) AND isnotnull(o_orderkey#X)) -(70) Exchange +(75) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(71) Scan parquet +(76) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(72) HashAggregate +(77) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(73) Exchange +(78) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(74) HashAggregate +(79) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(75) Filter +(80) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(76) Project +(81) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(77) ShuffledHashJoin +(82) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(78) Exchange +(83) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(79) ShuffledHashJoin +(84) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(80) Project +(85) Project Output [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_custkey#X, o_totalprice#X, o_orderdate#X] -(81) Exchange +(86) Exchange Input [5]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(82) Scan parquet +(87) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey)] ReadSchema: struct -(83) Filter +(88) Filter Input [2]: [l_orderkey#X, l_quantity#X] Condition : isnotnull(l_orderkey#X) -(84) Exchange +(89) Exchange Input [2]: [l_orderkey#X, l_quantity#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(90) Scan parquet Output [2]: [l_orderkey#X, l_quantity#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(86) HashAggregate +(91) HashAggregate Input [2]: [l_orderkey#X, l_quantity#X] Keys [1]: [l_orderkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [l_orderkey#X, sum#X, isEmpty#X] -(87) Exchange +(92) Exchange Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(88) HashAggregate +(93) HashAggregate Input [3]: [l_orderkey#X, sum#X, isEmpty#X] Keys [1]: [l_orderkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [2]: [l_orderkey#X, sum(l_quantity#X)#X AS sum(l_quantity#X)#X] -(89) Filter +(94) Filter Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] Condition : (isnotnull(sum(l_quantity#X)#X) AND (sum(l_quantity#X)#X > 300.00)) -(90) Project +(95) Project Output [1]: [l_orderkey#X] Input [2]: [l_orderkey#X, sum(l_quantity#X)#X] -(91) ShuffledHashJoin +(96) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(92) ShuffledHashJoin +(97) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(93) Project +(98) Project Output [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Input [7]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_orderkey#X, l_quantity#X] -(94) HashAggregate +(99) HashAggregate Input [6]: [c_custkey#X, c_name#X, o_orderkey#X, o_totalprice#X, o_orderdate#X, l_quantity#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] -(95) HashAggregate +(100) HashAggregate Input [7]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum#X, isEmpty#X] Keys [5]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity#X)#X AS sum(l_quantity)#X] -(96) TakeOrderedAndProject +(101) TakeOrderedAndProject Input [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: X, [o_totalprice#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] -(97) AdaptiveSparkPlan +(102) AdaptiveSparkPlan Output [6]: [c_name#X, c_custkey#X, o_orderkey#X, o_orderdate#X, o_totalprice#X, sum(l_quantity)#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt index 2f7bb7995dd9e..baf4b2a51607a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/19.txt @@ -1,35 +1,37 @@ == Physical Plan == -AdaptiveSparkPlan (34) +AdaptiveSparkPlan (36) +- == Final Plan == - VeloxColumnarToRowExec (22) - +- ^ RegularHashAggregateExecTransformer (20) - +- ^ RegularHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (24) + +- ^ RegularHashAggregateExecTransformer (22) + +- ^ RegularHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - HashAggregate (33) - +- HashAggregate (32) - +- Project (31) - +- ShuffledHashJoin Inner BuildRight (30) - :- Exchange (26) - : +- Project (25) - : +- Filter (24) - : +- Scan parquet (23) - +- Exchange (29) - +- Filter (28) - +- Scan parquet (27) + HashAggregate (35) + +- HashAggregate (34) + +- Project (33) + +- ShuffledHashJoin Inner BuildRight (32) + :- Exchange (28) + : +- Project (27) + : +- Filter (26) + : +- Scan parquet (25) + +- Exchange (31) + +- Filter (30) + +- Scan parquet (29) (1) Scan parquet @@ -51,142 +53,150 @@ Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipin Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X, p_brand#X, p_size#X, p_container#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(19) RegularHashAggregateExecTransformer +(21) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(20) RegularHashAggregateExecTransformer +(22) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(22) VeloxColumnarToRowExec +(24) VeloxColumnarToRowExec Input [1]: [revenue#X] -(23) Scan parquet +(25) Scan parquet Output [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipinstruct), In(l_shipmode, [AIR,AIR REG]), EqualTo(l_shipinstruct,DELIVER IN PERSON), IsNotNull(l_partkey), Or(Or(And(GreaterThanOrEqual(l_quantity,1.00),LessThanOrEqual(l_quantity,11.00)),And(GreaterThanOrEqual(l_quantity,10.00),LessThanOrEqual(l_quantity,20.00))),And(GreaterThanOrEqual(l_quantity,20.00),LessThanOrEqual(l_quantity,30.00)))] ReadSchema: struct -(24) Filter +(26) Filter Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] Condition : ((((isnotnull(l_shipinstruct#X) AND l_shipmode#X IN (AIR,AIR REG)) AND (l_shipinstruct#X = DELIVER IN PERSON)) AND isnotnull(l_partkey#X)) AND ((((l_quantity#X >= 1.00) AND (l_quantity#X <= 11.00)) OR ((l_quantity#X >= 10.00) AND (l_quantity#X <= 20.00))) OR ((l_quantity#X >= 20.00) AND (l_quantity#X <= 30.00)))) -(25) Project +(27) Project Output [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, l_shipinstruct#X, l_shipmode#X] -(26) Exchange +(28) Exchange Input [4]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(27) Scan parquet +(29) Scan parquet Output [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#X),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#X),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#X),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct -(28) Filter +(30) Filter Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Condition : (((isnotnull(p_size#X) AND (p_size#X >= 1)) AND isnotnull(p_partkey#X)) AND (((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#X <= 5)) OR (((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#X <= 10))) OR (((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#X <= 15)))) -(29) Exchange +(31) Exchange Input [4]: [p_partkey#X, p_brand#X, p_size#X, p_container#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(30) ShuffledHashJoin +(32) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: Inner Join condition: (((((((p_brand#X = Brand#X) AND p_container#X IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#X >= 1.00)) AND (l_quantity#X <= 11.00)) AND (p_size#X <= 5)) OR (((((p_brand#X = Brand#X) AND p_container#X IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#X >= 10.00)) AND (l_quantity#X <= 20.00)) AND (p_size#X <= 10))) OR (((((p_brand#X = Brand#X) AND p_container#X IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#X >= 20.00)) AND (l_quantity#X <= 30.00)) AND (p_size#X <= 15))) -(31) Project +(33) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [8]: [l_partkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, p_partkey#X, p_brand#X, p_size#X, p_container#X] -(32) HashAggregate +(34) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(33) HashAggregate +(35) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(34) AdaptiveSparkPlan +(36) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt index e8aa97a29e7a4..7ddecfe855eb3 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/20.txt @@ -1,109 +1,119 @@ == Physical Plan == -AdaptiveSparkPlan (123) +AdaptiveSparkPlan (133) +- == Final Plan == - VeloxColumnarToRowExec (83) - +- AQEShuffleRead (82) - +- ShuffleQueryStage (81), Statistics(X) - +- ColumnarExchange (80) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (62) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (55) - : :- ^ InputIteratorTransformer (31) - : : +- ShuffleQueryStage (29), Statistics(X) - : : +- ColumnarExchange (28) - : : +- ^ ProjectExecTransformer (26) - : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (25) - : : :- ^ InputIteratorTransformer (16) - : : : +- ShuffleQueryStage (14), Statistics(X) - : : : +- ColumnarExchange (13) - : : : +- ^ ProjectExecTransformer (11) - : : : +- ^ FilterExecTransformer (10) - : : : +- ^ Scan parquet (9) - : : +- ^ InputIteratorTransformer (24) - : : +- ShuffleQueryStage (22), Statistics(X) - : : +- ColumnarExchange (21) - : : +- ^ ProjectExecTransformer (19) - : : +- ^ FilterExecTransformer (18) - : : +- ^ Scan parquet (17) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ ProjectExecTransformer (47) - : +- ^ RegularHashAggregateExecTransformer (46) - : +- ^ RegularHashAggregateExecTransformer (45) - : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (44) - : :- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ FilterExecTransformer (33) - : : +- ^ Scan parquet (32) - : +- ^ InputIteratorTransformer (43) - : +- ShuffleQueryStage (41), Statistics(X) - : +- ReusedExchange (40) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (93) + +- AQEShuffleRead (92) + +- ShuffleQueryStage (91), Statistics(X) + +- ColumnarExchange (90) + +- VeloxAppendBatches (89) + +- ^ ProjectExecTransformer (87) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (86) + :- ^ InputIteratorTransformer (76) + : +- ShuffleQueryStage (74), Statistics(X) + : +- ColumnarExchange (73) + : +- VeloxAppendBatches (72) + : +- ^ ProjectExecTransformer (70) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (69) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (68) + : +- ShuffleQueryStage (66), Statistics(X) + : +- ColumnarExchange (65) + : +- VeloxAppendBatches (64) + : +- ^ ProjectExecTransformer (62) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (61) + : :- ^ InputIteratorTransformer (35) + : : +- ShuffleQueryStage (33), Statistics(X) + : : +- ColumnarExchange (32) + : : +- VeloxAppendBatches (31) + : : +- ^ ProjectExecTransformer (29) + : : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (28) + : : :- ^ InputIteratorTransformer (18) + : : : +- ShuffleQueryStage (16), Statistics(X) + : : : +- ColumnarExchange (15) + : : : +- VeloxAppendBatches (14) + : : : +- ^ ProjectExecTransformer (12) + : : : +- ^ FilterExecTransformer (11) + : : : +- ^ Scan parquet (10) + : : +- ^ InputIteratorTransformer (27) + : : +- ShuffleQueryStage (25), Statistics(X) + : : +- ColumnarExchange (24) + : : +- VeloxAppendBatches (23) + : : +- ^ ProjectExecTransformer (21) + : : +- ^ FilterExecTransformer (20) + : : +- ^ Scan parquet (19) + : +- ^ InputIteratorTransformer (60) + : +- ShuffleQueryStage (58), Statistics(X) + : +- ColumnarExchange (57) + : +- VeloxAppendBatches (56) + : +- ^ ProjectExecTransformer (54) + : +- ^ FilterExecTransformer (53) + : +- ^ ProjectExecTransformer (52) + : +- ^ RegularHashAggregateExecTransformer (51) + : +- ^ RegularHashAggregateExecTransformer (50) + : +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (49) + : :- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ FilterExecTransformer (37) + : : +- ^ Scan parquet (36) + : +- ^ InputIteratorTransformer (48) + : +- ShuffleQueryStage (46), Statistics(X) + : +- ReusedExchange (45) + +- ^ InputIteratorTransformer (85) + +- ShuffleQueryStage (83), Statistics(X) + +- ColumnarExchange (82) + +- VeloxAppendBatches (81) + +- ^ ProjectExecTransformer (79) + +- ^ FilterExecTransformer (78) + +- ^ Scan parquet (77) +- == Initial Plan == - Sort (122) - +- Exchange (121) - +- Project (120) - +- ShuffledHashJoin Inner BuildRight (119) - :- Exchange (114) - : +- Project (113) - : +- ShuffledHashJoin LeftSemi BuildRight (112) - : :- Exchange (86) - : : +- Filter (85) - : : +- Scan parquet (84) - : +- Exchange (111) - : +- Project (110) - : +- ShuffledHashJoin Inner BuildLeft (109) - : :- Exchange (95) - : : +- ShuffledHashJoin LeftSemi BuildRight (94) - : : :- Exchange (89) - : : : +- Filter (88) - : : : +- Scan parquet (87) - : : +- Exchange (93) - : : +- Project (92) - : : +- Filter (91) - : : +- Scan parquet (90) - : +- Exchange (108) - : +- Filter (107) - : +- HashAggregate (106) - : +- HashAggregate (105) - : +- ShuffledHashJoin LeftSemi BuildRight (104) - : :- Exchange (99) - : : +- Project (98) - : : +- Filter (97) - : : +- Scan parquet (96) - : +- Exchange (103) - : +- Project (102) - : +- Filter (101) - : +- Scan parquet (100) - +- Exchange (118) - +- Project (117) - +- Filter (116) - +- Scan parquet (115) + Sort (132) + +- Exchange (131) + +- Project (130) + +- ShuffledHashJoin Inner BuildRight (129) + :- Exchange (124) + : +- Project (123) + : +- ShuffledHashJoin LeftSemi BuildRight (122) + : :- Exchange (96) + : : +- Filter (95) + : : +- Scan parquet (94) + : +- Exchange (121) + : +- Project (120) + : +- ShuffledHashJoin Inner BuildLeft (119) + : :- Exchange (105) + : : +- ShuffledHashJoin LeftSemi BuildRight (104) + : : :- Exchange (99) + : : : +- Filter (98) + : : : +- Scan parquet (97) + : : +- Exchange (103) + : : +- Project (102) + : : +- Filter (101) + : : +- Scan parquet (100) + : +- Exchange (118) + : +- Filter (117) + : +- HashAggregate (116) + : +- HashAggregate (115) + : +- ShuffledHashJoin LeftSemi BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- Filter (107) + : : +- Scan parquet (106) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (128) + +- Project (127) + +- Filter (126) + +- Scan parquet (125) (1) Scan parquet @@ -125,518 +135,558 @@ Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [5]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(15) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(18) FilterExecTransformer +(20) FilterExecTransformer Input [2]: [p_partkey#X, p_name#X] Arguments: (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [2]: [hash(p_partkey#X, 42) AS hash_partition_key#X, p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(21) ColumnarExchange +(23) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(24) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(25) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(23) InputAdapter +(26) InputAdapter Input [1]: [p_partkey#X] -(24) InputIteratorTransformer +(27) InputIteratorTransformer Input [1]: [p_partkey#X] -(25) ShuffledHashJoinExecTransformer +(28) ShuffledHashJoinExecTransformer Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(ps_partkey#X, ps_suppkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_availqty#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] -(32) Scan parquet +(36) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(33) FilterExecTransformer +(37) FilterExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [4]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] +Arguments: X + +(41) ColumnarExchange Input [4]: [hash_partition_key#X, l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_partkey#X, l_suppkey#X, l_quantity#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] -(40) ReusedExchange [Reuses operator id: 21] +(45) ReusedExchange [Reuses operator id: 24] Output [1]: [p_partkey#X] -(41) ShuffleQueryStage +(46) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(42) InputAdapter +(47) InputAdapter Input [1]: [p_partkey#X] -(43) InputIteratorTransformer +(48) InputIteratorTransformer Input [1]: [p_partkey#X] -(44) ShuffledHashJoinExecTransformer +(49) ShuffledHashJoinExecTransformer Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(45) RegularHashAggregateExecTransformer +(50) RegularHashAggregateExecTransformer Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(46) RegularHashAggregateExecTransformer +(51) RegularHashAggregateExecTransformer Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(47) ProjectExecTransformer +(52) ProjectExecTransformer Output [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [l_partkey#X, l_suppkey#X, sum(l_quantity#X)#X] -(48) FilterExecTransformer +(53) FilterExecTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: isnotnull((0.5 * sum(l_quantity))#X) -(49) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(l_partkey#X, l_suppkey#X, 42) AS hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(50) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: false -(51) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: X -(53) InputAdapter +(59) InputAdapter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(54) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(55) ShuffledHashJoinExecTransformer +(61) ShuffledHashJoinExecTransformer Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(56) ProjectExecTransformer +(62) ProjectExecTransformer Output [2]: [hash(ps_suppkey#X, 42) AS hash_partition_key#X, ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(57) WholeStageCodegenTransformer (X) +(63) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: false -(58) ColumnarExchange +(64) VeloxAppendBatches +Input [2]: [hash_partition_key#X, ps_suppkey#X] +Arguments: X + +(65) ColumnarExchange Input [2]: [hash_partition_key#X, ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [ps_suppkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(66) ShuffleQueryStage Output [1]: [ps_suppkey#X] Arguments: X -(60) InputAdapter +(67) InputAdapter Input [1]: [ps_suppkey#X] -(61) InputIteratorTransformer +(68) InputIteratorTransformer Input [1]: [ps_suppkey#X] -(62) ShuffledHashJoinExecTransformer +(69) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(63) ProjectExecTransformer +(70) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(71) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: false -(65) ColumnarExchange +(72) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] +Arguments: X + +(73) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_address#X, s_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(74) ShuffleQueryStage Output [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: X -(67) InputAdapter +(75) InputAdapter Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(68) InputIteratorTransformer +(76) InputIteratorTransformer Input [3]: [s_name#X, s_address#X, s_nationkey#X] -(69) Scan parquet +(77) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(78) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(71) ProjectExecTransformer +(79) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(80) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(73) ColumnarExchange +(81) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(82) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(83) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(75) InputAdapter +(84) InputAdapter Input [1]: [n_nationkey#X] -(76) InputIteratorTransformer +(85) InputIteratorTransformer Input [1]: [n_nationkey#X] -(77) ShuffledHashJoinExecTransformer +(86) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(87) ProjectExecTransformer Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(79) WholeStageCodegenTransformer (X) +(88) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, s_address#X] Arguments: false -(80) ColumnarExchange +(89) VeloxAppendBatches +Input [2]: [s_name#X, s_address#X] +Arguments: X + +(90) ColumnarExchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(91) ShuffleQueryStage Output [2]: [s_name#X, s_address#X] Arguments: X -(82) AQEShuffleRead +(92) AQEShuffleRead Input [2]: [s_name#X, s_address#X] Arguments: local -(83) VeloxColumnarToRowExec +(93) VeloxColumnarToRowExec Input [2]: [s_name#X, s_address#X] -(84) Scan parquet +(94) Scan parquet Output [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_nationkey)] ReadSchema: struct -(85) Filter +(95) Filter Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Condition : isnotnull(s_nationkey#X) -(86) Exchange +(96) Exchange Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(87) Scan parquet +(97) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_availqty), IsNotNull(ps_partkey), IsNotNull(ps_suppkey)] ReadSchema: struct -(88) Filter +(98) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Condition : ((isnotnull(ps_availqty#X) AND isnotnull(ps_partkey#X)) AND isnotnull(ps_suppkey#X)) -(89) Exchange +(99) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(90) Scan parquet +(100) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(91) Filter +(101) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(92) Project +(102) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(93) Exchange +(103) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) ShuffledHashJoin +(104) ShuffledHashJoin Left keys [1]: [ps_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(95) Exchange +(105) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X] Arguments: hashpartitioning(ps_partkey#X, ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) Scan parquet +(106) Scan parquet Output [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), IsNotNull(l_partkey), IsNotNull(l_suppkey)] ReadSchema: struct -(97) Filter +(107) Filter Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND isnotnull(l_partkey#X)) AND isnotnull(l_suppkey#X)) -(98) Project +(108) Project Output [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Input [4]: [l_partkey#X, l_suppkey#X, l_quantity#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringStartsWith(p_name,forest)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [p_partkey#X, p_name#X] Condition : (isnotnull(p_name#X) AND StartsWith(p_name#X, forest)) -(102) Project +(112) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(103) Exchange +(113) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(104) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_partkey#X] Right keys [1]: [p_partkey#X] Join type: LeftSemi Join condition: None -(105) HashAggregate +(115) HashAggregate Input [3]: [l_partkey#X, l_suppkey#X, l_quantity#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [partial_sum(l_quantity#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] -(106) HashAggregate +(116) HashAggregate Input [4]: [l_partkey#X, l_suppkey#X, sum#X, isEmpty#X] Keys [2]: [l_partkey#X, l_suppkey#X] Functions [1]: [sum(l_quantity#X)] Aggregate Attributes [1]: [sum(l_quantity#X)#X] Results [3]: [(0.5 * sum(l_quantity#X)#X) AS (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(107) Filter +(117) Filter Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Condition : isnotnull((0.5 * sum(l_quantity))#X) -(108) Exchange +(118) Exchange Input [3]: [(0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] Arguments: hashpartitioning(l_partkey#X, l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [2]: [ps_partkey#X, ps_suppkey#X] Right keys [2]: [l_partkey#X, l_suppkey#X] Join type: Inner Join condition: (cast(ps_availqty#X as decimal(24,3)) > (0.5 * sum(l_quantity))#X) -(110) Project +(120) Project Output [1]: [ps_suppkey#X] Input [6]: [ps_partkey#X, ps_suppkey#X, ps_availqty#X, (0.5 * sum(l_quantity))#X, l_partkey#X, l_suppkey#X] -(111) Exchange +(121) Exchange Input [1]: [ps_suppkey#X] Arguments: hashpartitioning(ps_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(122) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [ps_suppkey#X] Join type: LeftSemi Join condition: None -(113) Project +(123) Project Output [3]: [s_name#X, s_address#X, s_nationkey#X] Input [4]: [s_suppkey#X, s_name#X, s_address#X, s_nationkey#X] -(114) Exchange +(124) Exchange Input [3]: [s_name#X, s_address#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) Scan parquet +(125) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,CANADA), IsNotNull(n_nationkey)] ReadSchema: struct -(116) Filter +(126) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = CANADA)) AND isnotnull(n_nationkey#X)) -(117) Project +(127) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(118) Exchange +(128) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(119) ShuffledHashJoin +(129) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(120) Project +(130) Project Output [2]: [s_name#X, s_address#X] Input [4]: [s_name#X, s_address#X, s_nationkey#X, n_nationkey#X] -(121) Exchange +(131) Exchange Input [2]: [s_name#X, s_address#X] Arguments: rangepartitioning(s_name#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(122) Sort +(132) Sort Input [2]: [s_name#X, s_address#X] Arguments: [s_name#X ASC NULLS FIRST], true, 0 -(123) AdaptiveSparkPlan +(133) AdaptiveSparkPlan Output [2]: [s_name#X, s_address#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt index 323db548d24e4..ede94728ceb23 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/21.txt @@ -1,103 +1,112 @@ == Physical Plan == -AdaptiveSparkPlan (118) +AdaptiveSparkPlan (127) +- == Final Plan == - VeloxColumnarToRowExec (81) - +- ^ RegularHashAggregateExecTransformer (79) - +- ^ InputIteratorTransformer (78) - +- ShuffleQueryStage (76), Statistics(X) - +- ColumnarExchange (75) - +- ^ ProjectExecTransformer (73) - +- ^ FlushableHashAggregateExecTransformer (72) - +- ^ ProjectExecTransformer (71) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) - :- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (55) - : :- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (40) - : : :- ^ InputIteratorTransformer (8) - : : : +- ShuffleQueryStage (6), Statistics(X) - : : : +- ColumnarExchange (5) - : : : +- ^ ProjectExecTransformer (3) - : : : +- ^ FilterExecTransformer (2) - : : : +- ^ Scan parquet (1) - : : +- ^ InputIteratorTransformer (39) - : : +- ShuffleQueryStage (37), Statistics(X) - : : +- ColumnarExchange (36) - : : +- ^ ProjectExecTransformer (34) - : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (33) - : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (24) - : : : :- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (23) - : : : +- ShuffleQueryStage (21), Statistics(X) - : : : +- ColumnarExchange (20) - : : : +- ^ ProjectExecTransformer (18) - : : : +- ^ Scan parquet (17) - : : +- ^ InputIteratorTransformer (32) - : : +- ShuffleQueryStage (30), Statistics(X) - : : +- ColumnarExchange (29) - : : +- ^ ProjectExecTransformer (27) - : : +- ^ FilterExecTransformer (26) - : : +- ^ Scan parquet (25) - : +- ^ InputIteratorTransformer (54) - : +- ShuffleQueryStage (52), Statistics(X) - : +- ColumnarExchange (51) - : +- ^ ProjectExecTransformer (49) - : +- ^ FilterExecTransformer (48) - : +- ^ Scan parquet (47) - +- ^ InputIteratorTransformer (69) - +- ShuffleQueryStage (67), Statistics(X) - +- ColumnarExchange (66) - +- ^ ProjectExecTransformer (64) - +- ^ FilterExecTransformer (63) - +- ^ Scan parquet (62) + VeloxColumnarToRowExec (90) + +- ^ RegularHashAggregateExecTransformer (88) + +- ^ InputIteratorTransformer (87) + +- ShuffleQueryStage (85), Statistics(X) + +- ColumnarExchange (84) + +- ^ ProjectExecTransformer (82) + +- ^ FlushableHashAggregateExecTransformer (81) + +- ^ ProjectExecTransformer (80) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (79) + :- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) + : :- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (45) + : : :- ^ InputIteratorTransformer (9) + : : : +- ShuffleQueryStage (7), Statistics(X) + : : : +- ColumnarExchange (6) + : : : +- VeloxAppendBatches (5) + : : : +- ^ ProjectExecTransformer (3) + : : : +- ^ FilterExecTransformer (2) + : : : +- ^ Scan parquet (1) + : : +- ^ InputIteratorTransformer (44) + : : +- ShuffleQueryStage (42), Statistics(X) + : : +- ColumnarExchange (41) + : : +- VeloxAppendBatches (40) + : : +- ^ ProjectExecTransformer (38) + : : +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (37) + : : :- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (27) + : : : :- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (26) + : : : +- ShuffleQueryStage (24), Statistics(X) + : : : +- ColumnarExchange (23) + : : : +- VeloxAppendBatches (22) + : : : +- ^ ProjectExecTransformer (20) + : : : +- ^ Scan parquet (19) + : : +- ^ InputIteratorTransformer (36) + : : +- ShuffleQueryStage (34), Statistics(X) + : : +- ColumnarExchange (33) + : : +- VeloxAppendBatches (32) + : : +- ^ ProjectExecTransformer (30) + : : +- ^ FilterExecTransformer (29) + : : +- ^ Scan parquet (28) + : +- ^ InputIteratorTransformer (61) + : +- ShuffleQueryStage (59), Statistics(X) + : +- ColumnarExchange (58) + : +- VeloxAppendBatches (57) + : +- ^ ProjectExecTransformer (55) + : +- ^ FilterExecTransformer (54) + : +- ^ Scan parquet (53) + +- ^ InputIteratorTransformer (78) + +- ShuffleQueryStage (76), Statistics(X) + +- ColumnarExchange (75) + +- VeloxAppendBatches (74) + +- ^ ProjectExecTransformer (72) + +- ^ FilterExecTransformer (71) + +- ^ Scan parquet (70) +- == Initial Plan == - TakeOrderedAndProject (117) - +- HashAggregate (116) - +- Exchange (115) - +- HashAggregate (114) - +- Project (113) - +- ShuffledHashJoin Inner BuildRight (112) - :- Exchange (107) - : +- Project (106) - : +- ShuffledHashJoin Inner BuildRight (105) - : :- Exchange (100) - : : +- Project (99) - : : +- ShuffledHashJoin Inner BuildLeft (98) - : : :- Exchange (84) - : : : +- Filter (83) - : : : +- Scan parquet (82) - : : +- Exchange (97) - : : +- ShuffledHashJoin LeftAnti BuildRight (96) - : : :- ShuffledHashJoin LeftSemi BuildRight (91) - : : : :- Exchange (88) - : : : : +- Project (87) - : : : : +- Filter (86) - : : : : +- Scan parquet (85) - : : : +- Exchange (90) - : : : +- Scan parquet (89) - : : +- Exchange (95) - : : +- Project (94) - : : +- Filter (93) - : : +- Scan parquet (92) - : +- Exchange (104) - : +- Project (103) - : +- Filter (102) - : +- Scan parquet (101) - +- Exchange (111) - +- Project (110) - +- Filter (109) - +- Scan parquet (108) + TakeOrderedAndProject (126) + +- HashAggregate (125) + +- Exchange (124) + +- HashAggregate (123) + +- Project (122) + +- ShuffledHashJoin Inner BuildRight (121) + :- Exchange (116) + : +- Project (115) + : +- ShuffledHashJoin Inner BuildRight (114) + : :- Exchange (109) + : : +- Project (108) + : : +- ShuffledHashJoin Inner BuildLeft (107) + : : :- Exchange (93) + : : : +- Filter (92) + : : : +- Scan parquet (91) + : : +- Exchange (106) + : : +- ShuffledHashJoin LeftAnti BuildRight (105) + : : :- ShuffledHashJoin LeftSemi BuildRight (100) + : : : :- Exchange (97) + : : : : +- Project (96) + : : : : +- Filter (95) + : : : : +- Scan parquet (94) + : : : +- Exchange (99) + : : : +- Scan parquet (98) + : : +- Exchange (104) + : : +- Project (103) + : : +- Filter (102) + : : +- Scan parquet (101) + : +- Exchange (113) + : +- Project (112) + : +- Filter (111) + : +- Scan parquet (110) + +- Exchange (120) + +- Project (119) + +- Filter (118) + +- Scan parquet (117) (1) Scan parquet @@ -119,500 +128,536 @@ Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(17) Scan parquet +(19) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(24) ShuffledHashJoinExecTransformer +(27) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(25) Scan parquet +(28) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(26) FilterExecTransformer +(29) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(27) ProjectExecTransformer +(30) ProjectExecTransformer Output [3]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(28) WholeStageCodegenTransformer (X) +(31) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(29) ColumnarExchange +(32) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(33) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(30) ShuffleQueryStage +(34) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(31) InputAdapter +(35) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(32) InputIteratorTransformer +(36) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(33) ShuffledHashJoinExecTransformer +(37) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(34) ProjectExecTransformer +(38) ProjectExecTransformer Output [3]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X] Input [2]: [l_orderkey#X, l_suppkey#X] -(35) WholeStageCodegenTransformer (X) +(39) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: false -(36) ColumnarExchange +(40) VeloxAppendBatches +Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] +Arguments: X + +(41) ColumnarExchange Input [3]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X], [plan_id=X], [id=#X] -(37) ShuffleQueryStage +(42) ShuffleQueryStage Output [2]: [l_orderkey#X, l_suppkey#X] Arguments: X -(38) InputAdapter +(43) InputAdapter Input [2]: [l_orderkey#X, l_suppkey#X] -(39) InputIteratorTransformer +(44) InputIteratorTransformer Input [2]: [l_orderkey#X, l_suppkey#X] -(40) ShuffledHashJoinExecTransformer +(45) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X, l_orderkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] -(47) Scan parquet +(53) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(48) FilterExecTransformer +(54) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderstatus#X] Arguments: ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(49) ProjectExecTransformer +(55) ProjectExecTransformer Output [2]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(50) WholeStageCodegenTransformer (X) +(56) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: false -(51) ColumnarExchange +(57) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_orderkey#X] +Arguments: X + +(58) ColumnarExchange Input [2]: [hash_partition_key#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X], [plan_id=X], [id=#X] -(52) ShuffleQueryStage +(59) ShuffleQueryStage Output [1]: [o_orderkey#X] Arguments: X -(53) InputAdapter +(60) InputAdapter Input [1]: [o_orderkey#X] -(54) InputIteratorTransformer +(61) InputIteratorTransformer Input [1]: [o_orderkey#X] -(55) ShuffledHashJoinExecTransformer +(62) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_name#X, s_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [s_name#X, s_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [s_name#X, s_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [s_name#X, s_nationkey#X] -(62) Scan parquet +(70) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(63) FilterExecTransformer +(71) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(64) ProjectExecTransformer +(72) ProjectExecTransformer Output [2]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(65) WholeStageCodegenTransformer (X) +(73) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: false -(66) ColumnarExchange +(74) VeloxAppendBatches +Input [2]: [hash_partition_key#X, n_nationkey#X] +Arguments: X + +(75) ColumnarExchange Input [2]: [hash_partition_key#X, n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X], [plan_id=X], [id=#X] -(67) ShuffleQueryStage +(76) ShuffleQueryStage Output [1]: [n_nationkey#X] Arguments: X -(68) InputAdapter +(77) InputAdapter Input [1]: [n_nationkey#X] -(69) InputIteratorTransformer +(78) InputIteratorTransformer Input [1]: [n_nationkey#X] -(70) ShuffledHashJoinExecTransformer +(79) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(72) FlushableHashAggregateExecTransformer +(81) FlushableHashAggregateExecTransformer Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(73) ProjectExecTransformer +(82) ProjectExecTransformer Output [3]: [hash(s_name#X, 42) AS hash_partition_key#X, s_name#X, count#X] Input [2]: [s_name#X, count#X] -(74) WholeStageCodegenTransformer (X) +(83) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: false -(75) ColumnarExchange +(84) ColumnarExchange Input [3]: [hash_partition_key#X, s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [s_name#X, count#X], [plan_id=X], [id=#X] -(76) ShuffleQueryStage +(85) ShuffleQueryStage Output [2]: [s_name#X, count#X] Arguments: X -(77) InputAdapter +(86) InputAdapter Input [2]: [s_name#X, count#X] -(78) InputIteratorTransformer +(87) InputIteratorTransformer Input [2]: [s_name#X, count#X] -(79) RegularHashAggregateExecTransformer +(88) RegularHashAggregateExecTransformer Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(80) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [2]: [s_name#X, numwait#X] Arguments: false -(81) VeloxColumnarToRowExec +(90) VeloxColumnarToRowExec Input [2]: [s_name#X, numwait#X] -(82) Scan parquet +(91) Scan parquet Output [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(83) Filter +(92) Filter Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(84) Exchange +(93) Exchange Input [3]: [s_suppkey#X, s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(85) Scan parquet +(94) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(86) Filter +(95) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(87) Project +(96) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(88) Exchange +(97) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(89) Scan parquet +(98) Scan parquet Output [2]: [l_orderkey#X, l_suppkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(90) Exchange +(99) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(91) ShuffledHashJoin +(100) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: NOT (l_suppkey#X = l_suppkey#X) -(92) Scan parquet +(101) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_receiptdate), IsNotNull(l_commitdate)] ReadSchema: struct -(93) Filter +(102) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_receiptdate#X) AND isnotnull(l_commitdate#X)) AND (l_receiptdate#X > l_commitdate#X)) -(94) Project +(103) Project Output [2]: [l_orderkey#X, l_suppkey#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_commitdate#X, l_receiptdate#X] -(95) Exchange +(104) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(96) ShuffledHashJoin +(105) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftAnti Join condition: NOT (l_suppkey#X = l_suppkey#X) -(97) Exchange +(106) Exchange Input [2]: [l_orderkey#X, l_suppkey#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(99) Project +(108) Project Output [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Input [5]: [s_suppkey#X, s_name#X, s_nationkey#X, l_orderkey#X, l_suppkey#X] -(100) Exchange +(109) Exchange Input [3]: [s_name#X, s_nationkey#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(101) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_orderstatus#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderstatus), EqualTo(o_orderstatus,F), IsNotNull(o_orderkey)] ReadSchema: struct -(102) Filter +(111) Filter Input [2]: [o_orderkey#X, o_orderstatus#X] Condition : ((isnotnull(o_orderstatus#X) AND (o_orderstatus#X = F)) AND isnotnull(o_orderkey#X)) -(103) Project +(112) Project Output [1]: [o_orderkey#X] Input [2]: [o_orderkey#X, o_orderstatus#X] -(104) Exchange +(113) Exchange Input [1]: [o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) ShuffledHashJoin +(114) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(106) Project +(115) Project Output [2]: [s_name#X, s_nationkey#X] Input [4]: [s_name#X, s_nationkey#X, l_orderkey#X, o_orderkey#X] -(107) Exchange +(116) Exchange Input [2]: [s_name#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) Scan parquet +(117) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_name), EqualTo(n_name,SAUDI ARABIA), IsNotNull(n_nationkey)] ReadSchema: struct -(109) Filter +(118) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : ((isnotnull(n_name#X) AND (n_name#X = SAUDI ARABIA)) AND isnotnull(n_nationkey#X)) -(110) Project +(119) Project Output [1]: [n_nationkey#X] Input [2]: [n_nationkey#X, n_name#X] -(111) Exchange +(120) Exchange Input [1]: [n_nationkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) ShuffledHashJoin +(121) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(113) Project +(122) Project Output [1]: [s_name#X] Input [3]: [s_name#X, s_nationkey#X, n_nationkey#X] -(114) HashAggregate +(123) HashAggregate Input [1]: [s_name#X] Keys [1]: [s_name#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [s_name#X, count#X] -(115) Exchange +(124) Exchange Input [2]: [s_name#X, count#X] Arguments: hashpartitioning(s_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(116) HashAggregate +(125) HashAggregate Input [2]: [s_name#X, count#X] Keys [1]: [s_name#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [s_name#X, count(1)#X AS numwait#X] -(117) TakeOrderedAndProject +(126) TakeOrderedAndProject Input [2]: [s_name#X, numwait#X] Arguments: X, [numwait#X DESC NULLS LAST, s_name#X ASC NULLS FIRST], [s_name#X, numwait#X] -(118) AdaptiveSparkPlan +(127) AdaptiveSparkPlan Output [2]: [s_name#X, numwait#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt index add978887b0ac..58bbd22fa13c8 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/22.txt @@ -1,43 +1,46 @@ == Physical Plan == -AdaptiveSparkPlan (46) +AdaptiveSparkPlan (49) +- == Final Plan == - VeloxColumnarToRowExec (33) - +- ^ SortExecTransformer (31) - +- ^ InputIteratorTransformer (30) - +- ShuffleQueryStage (28), Statistics(X) - +- ColumnarExchange (27) - +- ^ RegularHashAggregateExecTransformer (25) - +- ^ InputIteratorTransformer (24) - +- ShuffleQueryStage (22), Statistics(X) - +- ColumnarExchange (21) - +- ^ ProjectExecTransformer (19) - +- ^ FlushableHashAggregateExecTransformer (18) - +- ^ ProjectExecTransformer (17) - +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (16) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (15) - +- ShuffleQueryStage (13), Statistics(X) - +- ColumnarExchange (12) - +- ^ ProjectExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (36) + +- ^ SortExecTransformer (34) + +- ^ InputIteratorTransformer (33) + +- ShuffleQueryStage (31), Statistics(X) + +- ColumnarExchange (30) + +- VeloxAppendBatches (29) + +- ^ RegularHashAggregateExecTransformer (27) + +- ^ InputIteratorTransformer (26) + +- ShuffleQueryStage (24), Statistics(X) + +- ColumnarExchange (23) + +- ^ ProjectExecTransformer (21) + +- ^ FlushableHashAggregateExecTransformer (20) + +- ^ ProjectExecTransformer (19) + +- ^ ShuffledHashJoinExecTransformer LeftAnti BuildRight (18) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (17) + +- ShuffleQueryStage (15), Statistics(X) + +- ColumnarExchange (14) + +- VeloxAppendBatches (13) + +- ^ ProjectExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (45) - +- Exchange (44) - +- HashAggregate (43) - +- Exchange (42) - +- HashAggregate (41) - +- Project (40) - +- ShuffledHashJoin LeftAnti BuildRight (39) - :- Exchange (36) - : +- Filter (35) - : +- Scan parquet (34) - +- Exchange (38) - +- Scan parquet (37) + Sort (48) + +- Exchange (47) + +- HashAggregate (46) + +- Exchange (45) + +- HashAggregate (44) + +- Project (43) + +- ShuffledHashJoin LeftAnti BuildRight (42) + :- Exchange (39) + : +- Filter (38) + : +- Scan parquet (37) + +- Exchange (41) + +- Scan parquet (40) (1) Scan parquet @@ -59,318 +62,330 @@ Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] +Arguments: X + +(6) ColumnarExchange Input [4]: [hash_partition_key#X, c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_phone#X, c_acctbal#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(9) Scan parquet +(10) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(10) ProjectExecTransformer +(11) ProjectExecTransformer Output [2]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_custkey#X] Input [1]: [o_custkey#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: false -(12) ColumnarExchange +(13) VeloxAppendBatches +Input [2]: [hash_partition_key#X, o_custkey#X] +Arguments: X + +(14) ColumnarExchange Input [2]: [hash_partition_key#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_custkey#X], [plan_id=X], [id=#X] -(13) ShuffleQueryStage +(15) ShuffleQueryStage Output [1]: [o_custkey#X] Arguments: X -(14) InputAdapter +(16) InputAdapter Input [1]: [o_custkey#X] -(15) InputIteratorTransformer +(17) InputIteratorTransformer Input [1]: [o_custkey#X] -(16) ShuffledHashJoinExecTransformer +(18) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(17) ProjectExecTransformer +(19) ProjectExecTransformer Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(18) FlushableHashAggregateExecTransformer +(20) FlushableHashAggregateExecTransformer Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(19) ProjectExecTransformer +(21) ProjectExecTransformer Output [5]: [hash(cntrycode#X, 42) AS hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(20) WholeStageCodegenTransformer (X) +(22) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: false -(21) ColumnarExchange +(23) ColumnarExchange Input [5]: [hash_partition_key#X, cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [cntrycode#X, count#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(22) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: X -(23) InputAdapter +(25) InputAdapter Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(24) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(25) RegularHashAggregateExecTransformer +(27) RegularHashAggregateExecTransformer Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(26) WholeStageCodegenTransformer (X) +(28) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(27) ColumnarExchange +(29) VeloxAppendBatches +Input [3]: [cntrycode#X, numcust#X, totacctbal#X] +Arguments: X + +(30) ColumnarExchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(28) ShuffleQueryStage +(31) ShuffleQueryStage Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: X -(29) InputAdapter +(32) InputAdapter Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(30) InputIteratorTransformer +(33) InputIteratorTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(31) SortExecTransformer +(34) SortExecTransformer Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(32) WholeStageCodegenTransformer (X) +(35) WholeStageCodegenTransformer (X) Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: false -(33) VeloxColumnarToRowExec +(36) VeloxColumnarToRowExec Input [3]: [cntrycode#X, numcust#X, totacctbal#X] -(34) Scan parquet +(37) Scan parquet Output [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal)] ReadSchema: struct -(35) Filter +(38) Filter Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) AND (cast(c_acctbal#X as decimal(16,6)) > Subquery subquery#X, [id=#X])) -(36) Exchange +(39) Exchange Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(37) Scan parquet +(40) Scan parquet Output [1]: [o_custkey#X] Batched: true Location: InMemoryFileIndex [*] ReadSchema: struct -(38) Exchange +(41) Exchange Input [1]: [o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) ShuffledHashJoin +(42) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: LeftAnti Join condition: None -(40) Project +(43) Project Output [2]: [substring(c_phone#X, 1, 2) AS cntrycode#X, c_acctbal#X] Input [3]: [c_custkey#X, c_phone#X, c_acctbal#X] -(41) HashAggregate +(44) HashAggregate Input [2]: [cntrycode#X, c_acctbal#X] Keys [1]: [cntrycode#X] Functions [2]: [partial_count(1), partial_sum(c_acctbal#X)] Aggregate Attributes [3]: [count#X, sum#X, isEmpty#X] Results [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] -(42) Exchange +(45) Exchange Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Arguments: hashpartitioning(cntrycode#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) HashAggregate +(46) HashAggregate Input [4]: [cntrycode#X, count#X, sum#X, isEmpty#X] Keys [1]: [cntrycode#X] Functions [2]: [count(1), sum(c_acctbal#X)] Aggregate Attributes [2]: [count(1)#X, sum(c_acctbal#X)#X] Results [3]: [cntrycode#X, count(1)#X AS numcust#X, sum(c_acctbal#X)#X AS totacctbal#X] -(44) Exchange +(47) Exchange Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: rangepartitioning(cntrycode#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(45) Sort +(48) Sort Input [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: [cntrycode#X ASC NULLS FIRST], true, 0 -(46) AdaptiveSparkPlan +(49) AdaptiveSparkPlan Output [3]: [cntrycode#X, numcust#X, totacctbal#X] Arguments: isFinalPlan=true ===== Subqueries ===== Subquery:1 Hosting operator id = 2 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (61) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- ^ FlushableHashAggregateExecTransformer (53) + +- ^ ProjectExecTransformer (52) + +- ^ FilterExecTransformer (51) + +- ^ Scan parquet (50) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) + HashAggregate (67) + +- Exchange (66) + +- HashAggregate (65) + +- Project (64) + +- Filter (63) + +- Scan parquet (62) -(47) Scan parquet +(50) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(48) FilterExecTransformer +(51) FilterExecTransformer Input [2]: [c_phone#X, c_acctbal#X] Arguments: ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(49) ProjectExecTransformer +(52) ProjectExecTransformer Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(50) FlushableHashAggregateExecTransformer +(53) FlushableHashAggregateExecTransformer Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(51) WholeStageCodegenTransformer (X) +(54) WholeStageCodegenTransformer (X) Input [2]: [sum#X, count#X] Arguments: false -(52) ColumnarExchange +(55) ColumnarExchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(53) ShuffleQueryStage +(56) ShuffleQueryStage Output [2]: [sum#X, count#X] Arguments: X -(54) InputAdapter +(57) InputAdapter Input [2]: [sum#X, count#X] -(55) InputIteratorTransformer +(58) InputIteratorTransformer Input [2]: [sum#X, count#X] -(56) RegularHashAggregateExecTransformer +(59) RegularHashAggregateExecTransformer Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(57) WholeStageCodegenTransformer (X) +(60) WholeStageCodegenTransformer (X) Input [1]: [avg(c_acctbal)#X] Arguments: false -(58) VeloxColumnarToRowExec +(61) VeloxColumnarToRowExec Input [1]: [avg(c_acctbal)#X] -(59) Scan parquet +(62) Scan parquet Output [2]: [c_phone#X, c_acctbal#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_acctbal), GreaterThan(c_acctbal,0.00)] ReadSchema: struct -(60) Filter +(63) Filter Input [2]: [c_phone#X, c_acctbal#X] Condition : ((isnotnull(c_acctbal#X) AND (c_acctbal#X > 0.00)) AND substring(c_phone#X, 1, 2) IN (13,31,23,29,30,18,17)) -(61) Project +(64) Project Output [1]: [c_acctbal#X] Input [2]: [c_phone#X, c_acctbal#X] -(62) HashAggregate +(65) HashAggregate Input [1]: [c_acctbal#X] Keys: [] Functions [1]: [partial_avg(c_acctbal#X)] Aggregate Attributes [2]: [sum#X, count#X] Results [2]: [sum#X, count#X] -(63) Exchange +(66) Exchange Input [2]: [sum#X, count#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(64) HashAggregate +(67) HashAggregate Input [2]: [sum#X, count#X] Keys: [] Functions [1]: [avg(c_acctbal#X)] Aggregate Attributes [1]: [avg(c_acctbal#X)#X] Results [1]: [avg(c_acctbal#X)#X AS avg(c_acctbal)#X] -(65) AdaptiveSparkPlan +(68) AdaptiveSparkPlan Output [1]: [avg(c_acctbal)#X] Arguments: isFinalPlan=true Subquery:2 Hosting operator id = 1 Hosting Expression = Subquery subquery#X, [id=#X] -AdaptiveSparkPlan (65) +AdaptiveSparkPlan (68) +- == Final Plan == - VeloxColumnarToRowExec (58) - +- ^ RegularHashAggregateExecTransformer (56) - +- ^ InputIteratorTransformer (55) - +- ShuffleQueryStage (53), Statistics(X) - +- ColumnarExchange (52) - +- ^ FlushableHashAggregateExecTransformer (50) - +- ^ ProjectExecTransformer (49) - +- ^ FilterExecTransformer (48) - +- ^ Scan parquet (47) + VeloxColumnarToRowExec (61) + +- ^ RegularHashAggregateExecTransformer (59) + +- ^ InputIteratorTransformer (58) + +- ShuffleQueryStage (56), Statistics(X) + +- ColumnarExchange (55) + +- ^ FlushableHashAggregateExecTransformer (53) + +- ^ ProjectExecTransformer (52) + +- ^ FilterExecTransformer (51) + +- ^ Scan parquet (50) +- == Initial Plan == - HashAggregate (64) - +- Exchange (63) - +- HashAggregate (62) - +- Project (61) - +- Filter (60) - +- Scan parquet (59) \ No newline at end of file + HashAggregate (67) + +- Exchange (66) + +- HashAggregate (65) + +- Project (64) + +- Filter (63) + +- Scan parquet (62) \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt index 69dc65c58e21b..709a1700b5c76 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/3.txt @@ -1,56 +1,60 @@ == Physical Plan == -AdaptiveSparkPlan (59) +AdaptiveSparkPlan (63) +- == Final Plan == - VeloxColumnarToRowExec (39) - +- TakeOrderedAndProjectExecTransformer (38) - +- ^ ProjectExecTransformer (36) - +- ^ RegularHashAggregateExecTransformer (35) - +- ^ RegularHashAggregateExecTransformer (34) - +- ^ ProjectExecTransformer (33) - +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - :- ^ InputIteratorTransformer (23) - : +- ShuffleQueryStage (21), Statistics(X) - : +- ColumnarExchange (20) - : +- ^ ProjectExecTransformer (18) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : :- ^ InputIteratorTransformer (8) - : : +- ShuffleQueryStage (6), Statistics(X) - : : +- ColumnarExchange (5) - : : +- ^ ProjectExecTransformer (3) - : : +- ^ FilterExecTransformer (2) - : : +- ^ Scan parquet (1) - : +- ^ InputIteratorTransformer (16) - : +- ShuffleQueryStage (14), Statistics(X) - : +- ColumnarExchange (13) - : +- ^ ProjectExecTransformer (11) - : +- ^ FilterExecTransformer (10) - : +- ^ Scan parquet (9) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ ProjectExecTransformer (26) - +- ^ FilterExecTransformer (25) - +- ^ Scan parquet (24) + VeloxColumnarToRowExec (43) + +- TakeOrderedAndProjectExecTransformer (42) + +- ^ ProjectExecTransformer (40) + +- ^ RegularHashAggregateExecTransformer (39) + +- ^ RegularHashAggregateExecTransformer (38) + +- ^ ProjectExecTransformer (37) + +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + :- ^ InputIteratorTransformer (26) + : +- ShuffleQueryStage (24), Statistics(X) + : +- ColumnarExchange (23) + : +- VeloxAppendBatches (22) + : +- ^ ProjectExecTransformer (20) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : :- ^ InputIteratorTransformer (9) + : : +- ShuffleQueryStage (7), Statistics(X) + : : +- ColumnarExchange (6) + : : +- VeloxAppendBatches (5) + : : +- ^ ProjectExecTransformer (3) + : : +- ^ FilterExecTransformer (2) + : : +- ^ Scan parquet (1) + : +- ^ InputIteratorTransformer (18) + : +- ShuffleQueryStage (16), Statistics(X) + : +- ColumnarExchange (15) + : +- VeloxAppendBatches (14) + : +- ^ ProjectExecTransformer (12) + : +- ^ FilterExecTransformer (11) + : +- ^ Scan parquet (10) + +- ^ InputIteratorTransformer (35) + +- ShuffleQueryStage (33), Statistics(X) + +- ColumnarExchange (32) + +- VeloxAppendBatches (31) + +- ^ ProjectExecTransformer (29) + +- ^ FilterExecTransformer (28) + +- ^ Scan parquet (27) +- == Initial Plan == - TakeOrderedAndProject (58) - +- HashAggregate (57) - +- HashAggregate (56) - +- Project (55) - +- ShuffledHashJoin Inner BuildRight (54) - :- Exchange (49) - : +- Project (48) - : +- ShuffledHashJoin Inner BuildLeft (47) - : :- Exchange (43) - : : +- Project (42) - : : +- Filter (41) - : : +- Scan parquet (40) - : +- Exchange (46) - : +- Filter (45) - : +- Scan parquet (44) - +- Exchange (53) - +- Project (52) - +- Filter (51) - +- Scan parquet (50) + TakeOrderedAndProject (62) + +- HashAggregate (61) + +- HashAggregate (60) + +- Project (59) + +- ShuffledHashJoin Inner BuildRight (58) + :- Exchange (53) + : +- Project (52) + : +- ShuffledHashJoin Inner BuildLeft (51) + : :- Exchange (47) + : : +- Project (46) + : : +- Filter (45) + : : +- Scan parquet (44) + : +- Exchange (50) + : +- Filter (49) + : +- Scan parquet (48) + +- Exchange (57) + +- Project (56) + +- Filter (55) + +- Scan parquet (54) (1) Scan parquet @@ -72,248 +76,264 @@ Input [2]: [c_custkey#X, c_mktsegment#X] Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, c_custkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [c_custkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [c_custkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [c_custkey#X] -(9) Scan parquet +(10) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [5]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(15) ColumnarExchange Input [5]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] +Arguments: X + +(23) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X, o_shippriority#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [4]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [4]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(34) RegularHashAggregateExecTransformer +(38) RegularHashAggregateExecTransformer Input [6]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X, _pre_X#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(35) RegularHashAggregateExecTransformer +(39) RegularHashAggregateExecTransformer Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(36) ProjectExecTransformer +(40) ProjectExecTransformer Output [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] Input [4]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X] -(37) WholeStageCodegenTransformer (X) +(41) WholeStageCodegenTransformer (X) Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: false -(38) TakeOrderedAndProjectExecTransformer +(42) TakeOrderedAndProjectExecTransformer Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X], 0 -(39) VeloxColumnarToRowExec +(43) VeloxColumnarToRowExec Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(40) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_mktsegment#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_mktsegment), EqualTo(c_mktsegment,BUILDING), IsNotNull(c_custkey)] ReadSchema: struct -(41) Filter +(45) Filter Input [2]: [c_custkey#X, c_mktsegment#X] Condition : ((isnotnull(c_mktsegment#X) AND (c_mktsegment#X = BUILDING)) AND isnotnull(c_custkey#X)) -(42) Project +(46) Project Output [1]: [c_custkey#X] Input [2]: [c_custkey#X, c_mktsegment#X] -(43) Exchange +(47) Exchange Input [1]: [c_custkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(44) Scan parquet +(48) Scan parquet Output [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), LessThan(o_orderdate,1995-03-15), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(45) Filter +(49) Filter Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Condition : (((isnotnull(o_orderdate#X) AND (o_orderdate#X < 1995-03-15)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(46) Exchange +(50) Exchange Input [4]: [o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) ShuffledHashJoin +(51) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(48) Project +(52) Project Output [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Input [5]: [c_custkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X, o_shippriority#X] -(49) Exchange +(53) Exchange Input [3]: [o_orderkey#X, o_orderdate#X, o_shippriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(50) Scan parquet +(54) Scan parquet Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThan(l_shipdate,1995-03-15), IsNotNull(l_orderkey)] ReadSchema: struct -(51) Filter +(55) Filter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((isnotnull(l_shipdate#X) AND (l_shipdate#X > 1995-03-15)) AND isnotnull(l_orderkey#X)) -(52) Project +(56) Project Output [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(53) Exchange +(57) Exchange Input [3]: [l_orderkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(54) ShuffledHashJoin +(58) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(55) Project +(59) Project Output [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Input [6]: [o_orderkey#X, o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] -(56) HashAggregate +(60) HashAggregate Input [5]: [o_orderdate#X, o_shippriority#X, l_orderkey#X, l_extendedprice#X, l_discount#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] -(57) HashAggregate +(61) HashAggregate Input [5]: [l_orderkey#X, o_orderdate#X, o_shippriority#X, sum#X, isEmpty#X] Keys [3]: [l_orderkey#X, o_orderdate#X, o_shippriority#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [4]: [l_orderkey#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X, o_orderdate#X, o_shippriority#X] -(58) TakeOrderedAndProject +(62) TakeOrderedAndProject Input [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: X, [revenue#X DESC NULLS LAST, o_orderdate#X ASC NULLS FIRST], [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] -(59) AdaptiveSparkPlan +(63) AdaptiveSparkPlan Output [4]: [l_orderkey#X, revenue#X, o_orderdate#X, o_shippriority#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt index d7f42dd7b3519..ebf46078f68b9 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/4.txt @@ -1,47 +1,50 @@ == Physical Plan == -AdaptiveSparkPlan (50) +AdaptiveSparkPlan (53) +- == Final Plan == - VeloxColumnarToRowExec (34) - +- ^ SortExecTransformer (32) - +- ^ InputIteratorTransformer (31) - +- ShuffleQueryStage (29), Statistics(X) - +- ColumnarExchange (28) - +- ^ RegularHashAggregateExecTransformer (26) - +- ^ InputIteratorTransformer (25) - +- ShuffleQueryStage (23), Statistics(X) - +- ColumnarExchange (22) - +- ^ ProjectExecTransformer (20) - +- ^ FlushableHashAggregateExecTransformer (19) - +- ^ ProjectExecTransformer (18) - +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (17) - :- ^ InputIteratorTransformer (8) - : +- ShuffleQueryStage (6), Statistics(X) - : +- ColumnarExchange (5) - : +- ^ ProjectExecTransformer (3) - : +- ^ FilterExecTransformer (2) - : +- ^ Scan parquet (1) - +- ^ InputIteratorTransformer (16) - +- ShuffleQueryStage (14), Statistics(X) - +- ColumnarExchange (13) - +- ^ ProjectExecTransformer (11) - +- ^ FilterExecTransformer (10) - +- ^ Scan parquet (9) + VeloxColumnarToRowExec (37) + +- ^ SortExecTransformer (35) + +- ^ InputIteratorTransformer (34) + +- ShuffleQueryStage (32), Statistics(X) + +- ColumnarExchange (31) + +- VeloxAppendBatches (30) + +- ^ RegularHashAggregateExecTransformer (28) + +- ^ InputIteratorTransformer (27) + +- ShuffleQueryStage (25), Statistics(X) + +- ColumnarExchange (24) + +- ^ ProjectExecTransformer (22) + +- ^ FlushableHashAggregateExecTransformer (21) + +- ^ ProjectExecTransformer (20) + +- ^ ShuffledHashJoinExecTransformer LeftSemi BuildRight (19) + :- ^ InputIteratorTransformer (9) + : +- ShuffleQueryStage (7), Statistics(X) + : +- ColumnarExchange (6) + : +- VeloxAppendBatches (5) + : +- ^ ProjectExecTransformer (3) + : +- ^ FilterExecTransformer (2) + : +- ^ Scan parquet (1) + +- ^ InputIteratorTransformer (18) + +- ShuffleQueryStage (16), Statistics(X) + +- ColumnarExchange (15) + +- VeloxAppendBatches (14) + +- ^ ProjectExecTransformer (12) + +- ^ FilterExecTransformer (11) + +- ^ Scan parquet (10) +- == Initial Plan == - Sort (49) - +- Exchange (48) - +- HashAggregate (47) - +- Exchange (46) - +- HashAggregate (45) - +- Project (44) - +- ShuffledHashJoin LeftSemi BuildRight (43) - :- Exchange (38) - : +- Project (37) - : +- Filter (36) - : +- Scan parquet (35) - +- Exchange (42) - +- Project (41) - +- Filter (40) - +- Scan parquet (39) + Sort (52) + +- Exchange (51) + +- HashAggregate (50) + +- Exchange (49) + +- HashAggregate (48) + +- Project (47) + +- ShuffledHashJoin LeftSemi BuildRight (46) + :- Exchange (41) + : +- Project (40) + : +- Filter (39) + : +- Scan parquet (38) + +- Exchange (45) + +- Project (44) + +- Filter (43) + +- Scan parquet (42) (1) Scan parquet @@ -63,202 +66,214 @@ Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderpriority#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderpriority#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [o_orderkey#X, o_orderpriority#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderpriority#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Arguments: ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [2]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [2]: [hash_partition_key#X, l_orderkey#X] +Arguments: X + +(15) ColumnarExchange Input [2]: [hash_partition_key#X, l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [1]: [l_orderkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [1]: [l_orderkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [1]: [l_orderkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(19) FlushableHashAggregateExecTransformer +(21) FlushableHashAggregateExecTransformer Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(20) ProjectExecTransformer +(22) ProjectExecTransformer Output [3]: [hash(o_orderpriority#X, 42) AS hash_partition_key#X, o_orderpriority#X, count#X] Input [2]: [o_orderpriority#X, count#X] -(21) WholeStageCodegenTransformer (X) +(23) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: false -(22) ColumnarExchange +(24) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [o_orderpriority#X, count#X], [plan_id=X], [id=#X] -(23) ShuffleQueryStage +(25) ShuffleQueryStage Output [2]: [o_orderpriority#X, count#X] Arguments: X -(24) InputAdapter +(26) InputAdapter Input [2]: [o_orderpriority#X, count#X] -(25) InputIteratorTransformer +(27) InputIteratorTransformer Input [2]: [o_orderpriority#X, count#X] -(26) RegularHashAggregateExecTransformer +(28) RegularHashAggregateExecTransformer Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(27) WholeStageCodegenTransformer (X) +(29) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(28) ColumnarExchange +(30) VeloxAppendBatches +Input [2]: [o_orderpriority#X, order_count#X] +Arguments: X + +(31) ColumnarExchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(32) ShuffleQueryStage Output [2]: [o_orderpriority#X, order_count#X] Arguments: X -(30) InputAdapter +(33) InputAdapter Input [2]: [o_orderpriority#X, order_count#X] -(31) InputIteratorTransformer +(34) InputIteratorTransformer Input [2]: [o_orderpriority#X, order_count#X] -(32) SortExecTransformer +(35) SortExecTransformer Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(33) WholeStageCodegenTransformer (X) +(36) WholeStageCodegenTransformer (X) Input [2]: [o_orderpriority#X, order_count#X] Arguments: false -(34) VeloxColumnarToRowExec +(37) VeloxColumnarToRowExec Input [2]: [o_orderpriority#X, order_count#X] -(35) Scan parquet +(38) Scan parquet Output [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1993-07-01), LessThan(o_orderdate,1993-10-01)] ReadSchema: struct -(36) Filter +(39) Filter Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] Condition : ((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1993-07-01)) AND (o_orderdate#X < 1993-10-01)) -(37) Project +(40) Project Output [2]: [o_orderkey#X, o_orderpriority#X] Input [3]: [o_orderkey#X, o_orderdate#X, o_orderpriority#X] -(38) Exchange +(41) Exchange Input [2]: [o_orderkey#X, o_orderpriority#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(39) Scan parquet +(42) Scan parquet Output [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_commitdate), IsNotNull(l_receiptdate)] ReadSchema: struct -(40) Filter +(43) Filter Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] Condition : ((isnotnull(l_commitdate#X) AND isnotnull(l_receiptdate#X)) AND (l_commitdate#X < l_receiptdate#X)) -(41) Project +(44) Project Output [1]: [l_orderkey#X] Input [3]: [l_orderkey#X, l_commitdate#X, l_receiptdate#X] -(42) Exchange +(45) Exchange Input [1]: [l_orderkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(43) ShuffledHashJoin +(46) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: LeftSemi Join condition: None -(44) Project +(47) Project Output [1]: [o_orderpriority#X] Input [2]: [o_orderkey#X, o_orderpriority#X] -(45) HashAggregate +(48) HashAggregate Input [1]: [o_orderpriority#X] Keys [1]: [o_orderpriority#X] Functions [1]: [partial_count(1)] Aggregate Attributes [1]: [count#X] Results [2]: [o_orderpriority#X, count#X] -(46) Exchange +(49) Exchange Input [2]: [o_orderpriority#X, count#X] Arguments: hashpartitioning(o_orderpriority#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(47) HashAggregate +(50) HashAggregate Input [2]: [o_orderpriority#X, count#X] Keys [1]: [o_orderpriority#X] Functions [1]: [count(1)] Aggregate Attributes [1]: [count(1)#X] Results [2]: [o_orderpriority#X, count(1)#X AS order_count#X] -(48) Exchange +(51) Exchange Input [2]: [o_orderpriority#X, order_count#X] Arguments: rangepartitioning(o_orderpriority#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(49) Sort +(52) Sort Input [2]: [o_orderpriority#X, order_count#X] Arguments: [o_orderpriority#X ASC NULLS FIRST], true, 0 -(50) AdaptiveSparkPlan +(53) AdaptiveSparkPlan Output [2]: [o_orderpriority#X, order_count#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt index 129b02c9548ea..8c3258e616f5f 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/5.txt @@ -1,115 +1,126 @@ == Physical Plan == -AdaptiveSparkPlan (134) +AdaptiveSparkPlan (145) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (133) - +- Exchange (132) - +- HashAggregate (131) - +- Exchange (130) - +- HashAggregate (129) - +- Project (128) - +- ShuffledHashJoin Inner BuildRight (127) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Project (100) - : : : : +- Filter (99) - : : : : +- Scan parquet (98) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (126) - +- Project (125) - +- Filter (124) - +- Scan parquet (123) + Sort (144) + +- Exchange (143) + +- HashAggregate (142) + +- Exchange (141) + +- HashAggregate (140) + +- Project (139) + +- ShuffledHashJoin Inner BuildRight (138) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Project (111) + : : : : +- Filter (110) + : : : : +- Scan parquet (109) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (137) + +- Project (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -131,562 +142,606 @@ Input [2]: [c_custkey#X, c_nationkey#X] Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [3]: [hash(o_custkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(15) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] +Arguments: X + +(23) ColumnarExchange Input [3]: [hash_partition_key#X, c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, o_orderkey#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [2]: [c_nationkey#X, o_orderkey#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [2]: [c_nationkey#X, o_orderkey#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [2]: [c_nationkey#X, o_orderkey#X] -(24) Scan parquet +(27) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(32) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, c_nationkey#X, 42) AS hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, s_nationkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [4]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [4]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [4]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] +Arguments: X + +(66) ColumnarExchange Input [4]: [hash_partition_key#X, n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [5]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] +Arguments: X + +(74) ColumnarExchange Input [5]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [1]: [r_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [1]: [r_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, (l_extendedprice#X * (1 - l_discount#X)) AS _pre_X#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, _pre_X#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum(_pre_X#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [4]: [hash(n_name#X, 42) AS hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Input [3]: [n_name#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [4]: [hash_partition_key#X, n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [n_name#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [3]: [n_name#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [3]: [n_name#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [2]: [n_name#X, revenue#X] +Arguments: X + +(99) ColumnarExchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [2]: [n_name#X, revenue#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [2]: [n_name#X, revenue#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [2]: [n_name#X, revenue#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [2]: [n_name#X, revenue#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [2]: [n_name#X, revenue#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(97) Exchange +(108) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(98) Scan parquet +(109) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1994-01-01), LessThan(o_orderdate,1995-01-01), IsNotNull(o_custkey), IsNotNull(o_orderkey)] ReadSchema: struct -(99) Filter +(110) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1994-01-01)) AND (o_orderdate#X < 1995-01-01)) AND isnotnull(o_custkey#X)) AND isnotnull(o_orderkey#X)) -(100) Project +(111) Project Output [2]: [o_orderkey#X, o_custkey#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(101) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [c_custkey#X] Right keys [1]: [o_custkey#X] Join type: Inner Join condition: None -(103) Project +(114) Project Output [2]: [c_nationkey#X, o_orderkey#X] Input [4]: [c_custkey#X, c_nationkey#X, o_orderkey#X, o_custkey#X] -(104) Exchange +(115) Exchange Input [2]: [c_nationkey#X, o_orderkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_orderkey), IsNotNull(l_suppkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : (isnotnull(l_orderkey#X) AND isnotnull(l_suppkey#X)) -(107) Exchange +(118) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_orderkey#X] Right keys [1]: [l_orderkey#X] Join type: Inner Join condition: None -(109) Project +(120) Project Output [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [c_nationkey#X, o_orderkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(110) Exchange +(121) Exchange Input [4]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(113) Exchange +(124) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, c_nationkey#X] Right keys [2]: [s_suppkey#X, s_nationkey#X] Join type: Inner Join condition: None -(115) Project +(126) Project Output [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [c_nationkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(116) Exchange +(127) Exchange Input [3]: [l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(119) Exchange +(130) Exchange Input [3]: [n_nationkey#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(121) Project +(132) Project Output [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Input [6]: [l_extendedprice#X, l_discount#X, s_nationkey#X, n_nationkey#X, n_name#X, n_regionkey#X] -(122) Exchange +(133) Exchange Input [4]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,ASIA), IsNotNull(r_regionkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = ASIA)) AND isnotnull(r_regionkey#X)) -(125) Project +(136) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(126) Exchange +(137) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) ShuffledHashJoin +(138) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(128) Project +(139) Project Output [3]: [l_extendedprice#X, l_discount#X, n_name#X] Input [5]: [l_extendedprice#X, l_discount#X, n_name#X, n_regionkey#X, r_regionkey#X] -(129) HashAggregate +(140) HashAggregate Input [3]: [l_extendedprice#X, l_discount#X, n_name#X] Keys [1]: [n_name#X] Functions [1]: [partial_sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [3]: [n_name#X, sum#X, isEmpty#X] -(130) Exchange +(141) Exchange Input [3]: [n_name#X, sum#X, isEmpty#X] Arguments: hashpartitioning(n_name#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(131) HashAggregate +(142) HashAggregate Input [3]: [n_name#X, sum#X, isEmpty#X] Keys [1]: [n_name#X] Functions [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))] Aggregate Attributes [1]: [sum((l_extendedprice#X * (1 - l_discount#X)))#X] Results [2]: [n_name#X, sum((l_extendedprice#X * (1 - l_discount#X)))#X AS revenue#X] -(132) Exchange +(143) Exchange Input [2]: [n_name#X, revenue#X] Arguments: rangepartitioning(revenue#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) Sort +(144) Sort Input [2]: [n_name#X, revenue#X] Arguments: [revenue#X DESC NULLS LAST], true, 0 -(134) AdaptiveSparkPlan +(145) AdaptiveSparkPlan Output [2]: [n_name#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt index 12d6c3ea85e4e..415d85f46ee56 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/6.txt @@ -1,22 +1,23 @@ == Physical Plan == -AdaptiveSparkPlan (19) +AdaptiveSparkPlan (20) +- == Final Plan == - VeloxColumnarToRowExec (12) - +- ^ RegularHashAggregateExecTransformer (10) - +- ^ InputIteratorTransformer (9) - +- ShuffleQueryStage (7), Statistics(X) - +- ColumnarExchange (6) - +- ^ FlushableHashAggregateExecTransformer (4) - +- ^ ProjectExecTransformer (3) - +- ^ FilterExecTransformer (2) - +- ^ Scan parquet (1) + VeloxColumnarToRowExec (13) + +- ^ RegularHashAggregateExecTransformer (11) + +- ^ InputIteratorTransformer (10) + +- ShuffleQueryStage (8), Statistics(X) + +- ColumnarExchange (7) + +- VeloxAppendBatches (6) + +- ^ RegularHashAggregateExecTransformer (4) + +- ^ ProjectExecTransformer (3) + +- ^ FilterExecTransformer (2) + +- ^ Scan parquet (1) +- == Initial Plan == - HashAggregate (18) - +- Exchange (17) - +- HashAggregate (16) - +- Project (15) - +- Filter (14) - +- Scan parquet (13) + HashAggregate (19) + +- Exchange (18) + +- HashAggregate (17) + +- Project (16) + +- Filter (15) + +- Scan parquet (14) (1) Scan parquet @@ -34,7 +35,7 @@ Arguments: (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnot Output [3]: [l_extendedprice#X, l_discount#X, (l_extendedprice#X * l_discount#X) AS _pre_X#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(4) FlushableHashAggregateExecTransformer +(4) RegularHashAggregateExecTransformer Input [3]: [l_extendedprice#X, l_discount#X, _pre_X#X] Keys: [] Functions [1]: [partial_sum(_pre_X#X)] @@ -45,67 +46,71 @@ Results [2]: [sum#X, isEmpty#X] Input [2]: [sum#X, isEmpty#X] Arguments: false -(6) ColumnarExchange +(6) VeloxAppendBatches +Input [2]: [sum#X, isEmpty#X] +Arguments: X + +(7) ColumnarExchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(7) ShuffleQueryStage +(8) ShuffleQueryStage Output [2]: [sum#X, isEmpty#X] Arguments: X -(8) InputAdapter +(9) InputAdapter Input [2]: [sum#X, isEmpty#X] -(9) InputIteratorTransformer +(10) InputIteratorTransformer Input [2]: [sum#X, isEmpty#X] -(10) RegularHashAggregateExecTransformer +(11) RegularHashAggregateExecTransformer Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(11) WholeStageCodegenTransformer (X) +(12) WholeStageCodegenTransformer (X) Input [1]: [revenue#X] Arguments: false -(12) VeloxColumnarToRowExec +(13) VeloxColumnarToRowExec Input [1]: [revenue#X] -(13) Scan parquet +(14) Scan parquet Output [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), IsNotNull(l_discount), IsNotNull(l_quantity), GreaterThanOrEqual(l_shipdate,1994-01-01), LessThan(l_shipdate,1995-01-01), GreaterThanOrEqual(l_discount,0.05), LessThanOrEqual(l_discount,0.07), LessThan(l_quantity,24.00)] ReadSchema: struct -(14) Filter +(15) Filter Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : (((((((isnotnull(l_shipdate#X) AND isnotnull(l_discount#X)) AND isnotnull(l_quantity#X)) AND (l_shipdate#X >= 1994-01-01)) AND (l_shipdate#X < 1995-01-01)) AND (l_discount#X >= 0.05)) AND (l_discount#X <= 0.07)) AND (l_quantity#X < 24.00)) -(15) Project +(16) Project Output [2]: [l_extendedprice#X, l_discount#X] Input [4]: [l_quantity#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) HashAggregate +(17) HashAggregate Input [2]: [l_extendedprice#X, l_discount#X] Keys: [] Functions [1]: [partial_sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [2]: [sum#X, isEmpty#X] -(17) Exchange +(18) Exchange Input [2]: [sum#X, isEmpty#X] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=X] -(18) HashAggregate +(19) HashAggregate Input [2]: [sum#X, isEmpty#X] Keys: [] Functions [1]: [sum((l_extendedprice#X * l_discount#X))] Aggregate Attributes [1]: [sum((l_extendedprice#X * l_discount#X))#X] Results [1]: [sum((l_extendedprice#X * l_discount#X))#X AS revenue#X] -(19) AdaptiveSparkPlan +(20) AdaptiveSparkPlan Output [1]: [revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt index 1e6af0683a390..fd60cdc1f478a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/7.txt @@ -1,110 +1,120 @@ == Physical Plan == -AdaptiveSparkPlan (128) +AdaptiveSparkPlan (138) +- == Final Plan == - VeloxColumnarToRowExec (90) - +- ^ SortExecTransformer (88) - +- ^ InputIteratorTransformer (87) - +- ShuffleQueryStage (85), Statistics(X) - +- ColumnarExchange (84) - +- ^ RegularHashAggregateExecTransformer (82) - +- ^ InputIteratorTransformer (81) - +- ShuffleQueryStage (79), Statistics(X) - +- ColumnarExchange (78) - +- ^ ProjectExecTransformer (76) - +- ^ FlushableHashAggregateExecTransformer (75) - +- ^ ProjectExecTransformer (74) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (73) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (72) - +- ShuffleQueryStage (70), Statistics(X) - +- ReusedExchange (69) + VeloxColumnarToRowExec (100) + +- ^ SortExecTransformer (98) + +- ^ InputIteratorTransformer (97) + +- ShuffleQueryStage (95), Statistics(X) + +- ColumnarExchange (94) + +- VeloxAppendBatches (93) + +- ^ RegularHashAggregateExecTransformer (91) + +- ^ InputIteratorTransformer (90) + +- ShuffleQueryStage (88), Statistics(X) + +- ColumnarExchange (87) + +- ^ ProjectExecTransformer (85) + +- ^ FlushableHashAggregateExecTransformer (84) + +- ^ ProjectExecTransformer (83) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (82) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (81) + +- ShuffleQueryStage (79), Statistics(X) + +- ReusedExchange (78) +- == Initial Plan == - Sort (127) - +- Exchange (126) - +- HashAggregate (125) - +- Exchange (124) - +- HashAggregate (123) - +- Project (122) - +- ShuffledHashJoin Inner BuildRight (121) - :- Exchange (117) - : +- Project (116) - : +- ShuffledHashJoin Inner BuildRight (115) - : :- Exchange (111) - : : +- Project (110) - : : +- ShuffledHashJoin Inner BuildRight (109) - : : :- Exchange (105) - : : : +- Project (104) - : : : +- ShuffledHashJoin Inner BuildRight (103) - : : : :- Exchange (99) - : : : : +- Project (98) - : : : : +- ShuffledHashJoin Inner BuildLeft (97) - : : : : :- Exchange (93) - : : : : : +- Filter (92) - : : : : : +- Scan parquet (91) - : : : : +- Exchange (96) - : : : : +- Filter (95) - : : : : +- Scan parquet (94) - : : : +- Exchange (102) - : : : +- Filter (101) - : : : +- Scan parquet (100) - : : +- Exchange (108) - : : +- Filter (107) - : : +- Scan parquet (106) - : +- Exchange (114) - : +- Filter (113) - : +- Scan parquet (112) - +- Exchange (120) - +- Filter (119) - +- Scan parquet (118) + Sort (137) + +- Exchange (136) + +- HashAggregate (135) + +- Exchange (134) + +- HashAggregate (133) + +- Project (132) + +- ShuffledHashJoin Inner BuildRight (131) + :- Exchange (127) + : +- Project (126) + : +- ShuffledHashJoin Inner BuildRight (125) + : :- Exchange (121) + : : +- Project (120) + : : +- ShuffledHashJoin Inner BuildRight (119) + : : :- Exchange (115) + : : : +- Project (114) + : : : +- ShuffledHashJoin Inner BuildRight (113) + : : : :- Exchange (109) + : : : : +- Project (108) + : : : : +- ShuffledHashJoin Inner BuildLeft (107) + : : : : :- Exchange (103) + : : : : : +- Filter (102) + : : : : : +- Scan parquet (101) + : : : : +- Exchange (106) + : : : : +- Filter (105) + : : : : +- Scan parquet (104) + : : : +- Exchange (112) + : : : +- Filter (111) + : : : +- Scan parquet (110) + : : +- Exchange (118) + : : +- Filter (117) + : : +- Scan parquet (116) + : +- Exchange (124) + : +- Filter (123) + : +- Scan parquet (122) + +- Exchange (130) + +- Filter (129) + +- Scan parquet (128) (1) Scan parquet @@ -126,534 +136,574 @@ Input [2]: [s_suppkey#X, s_nationkey#X] Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(6) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [6]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] +Arguments: X + +(23) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [o_orderkey#X, o_custkey#X] Arguments: (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X] Input [2]: [o_orderkey#X, o_custkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [o_orderkey#X, o_custkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [o_orderkey#X, o_custkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [o_orderkey#X, o_custkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] +Arguments: X + +(40) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] -(39) Scan parquet +(44) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(49) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] -(69) ReusedExchange [Reuses operator id: 58] +(78) ReusedExchange [Reuses operator id: 66] Output [2]: [n_nationkey#X, n_name#X] -(70) ShuffleQueryStage +(79) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(71) InputAdapter +(80) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(72) InputIteratorTransformer +(81) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(73) ShuffledHashJoinExecTransformer +(82) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(74) ProjectExecTransformer +(83) ProjectExecTransformer Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(75) FlushableHashAggregateExecTransformer +(84) FlushableHashAggregateExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(76) ProjectExecTransformer +(85) ProjectExecTransformer Output [6]: [hash(supp_nation#X, cust_nation#X, l_year#X, 42) AS hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(77) WholeStageCodegenTransformer (X) +(86) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: false -(78) ColumnarExchange +(87) ColumnarExchange Input [6]: [hash_partition_key#X, supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(79) ShuffleQueryStage +(88) ShuffleQueryStage Output [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: X -(80) InputAdapter +(89) InputAdapter Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(81) InputIteratorTransformer +(90) InputIteratorTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(82) RegularHashAggregateExecTransformer +(91) RegularHashAggregateExecTransformer Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(83) WholeStageCodegenTransformer (X) +(92) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(84) ColumnarExchange +(93) VeloxAppendBatches +Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] +Arguments: X + +(94) ColumnarExchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(85) ShuffleQueryStage +(95) ShuffleQueryStage Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: X -(86) InputAdapter +(96) InputAdapter Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(87) InputIteratorTransformer +(97) InputIteratorTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(88) SortExecTransformer +(98) SortExecTransformer Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(89) WholeStageCodegenTransformer (X) +(99) WholeStageCodegenTransformer (X) Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: false -(90) VeloxColumnarToRowExec +(100) VeloxColumnarToRowExec Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] -(91) Scan parquet +(101) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(92) Filter +(102) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(93) Exchange +(103) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(94) Scan parquet +(104) Scan parquet Output [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_shipdate), GreaterThanOrEqual(l_shipdate,1995-01-01), LessThanOrEqual(l_shipdate,1996-12-31), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(95) Filter +(105) Filter Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Condition : ((((isnotnull(l_shipdate#X) AND (l_shipdate#X >= 1995-01-01)) AND (l_shipdate#X <= 1996-12-31)) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(96) Exchange +(106) Exchange Input [5]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(97) ShuffledHashJoin +(107) ShuffledHashJoin Left keys [1]: [s_suppkey#X] Right keys [1]: [l_suppkey#X] Join type: Inner Join condition: None -(98) Project +(108) Project Output [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Input [7]: [s_suppkey#X, s_nationkey#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] -(99) Exchange +(109) Exchange Input [5]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(100) Scan parquet +(110) Scan parquet Output [2]: [o_orderkey#X, o_custkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(101) Filter +(111) Filter Input [2]: [o_orderkey#X, o_custkey#X] Condition : (isnotnull(o_orderkey#X) AND isnotnull(o_custkey#X)) -(102) Exchange +(112) Exchange Input [2]: [o_orderkey#X, o_custkey#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(103) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(104) Project +(114) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Input [7]: [s_nationkey#X, l_orderkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_orderkey#X, o_custkey#X] -(105) Exchange +(115) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(106) Scan parquet +(116) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(107) Filter +(117) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(108) Exchange +(118) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(109) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(110) Project +(120) Project Output [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, o_custkey#X, c_custkey#X, c_nationkey#X] -(111) Exchange +(121) Exchange Input [5]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(112) Scan parquet +(122) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,FRANCE),EqualTo(n_name,GERMANY))] ReadSchema: struct -(113) Filter +(123) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = FRANCE) OR (n_name#X = GERMANY))) -(114) Exchange +(124) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(115) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(116) Project +(126) Project Output [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Input [7]: [s_nationkey#X, l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_nationkey#X, n_name#X] -(117) Exchange +(127) Exchange Input [5]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(118) Scan parquet +(128) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), Or(EqualTo(n_name,GERMANY),EqualTo(n_name,FRANCE))] ReadSchema: struct -(119) Filter +(129) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : (isnotnull(n_nationkey#X) AND ((n_name#X = GERMANY) OR (n_name#X = FRANCE))) -(120) Exchange +(130) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(121) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: (((n_name#X = FRANCE) AND (n_name#X = GERMANY)) OR ((n_name#X = GERMANY) AND (n_name#X = FRANCE))) -(122) Project +(132) Project Output [4]: [n_name#X AS supp_nation#X, n_name#X AS cust_nation#X, year(l_shipdate#X) AS l_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X] Input [7]: [l_extendedprice#X, l_discount#X, l_shipdate#X, c_nationkey#X, n_name#X, n_nationkey#X, n_name#X] -(123) HashAggregate +(133) HashAggregate Input [4]: [supp_nation#X, cust_nation#X, l_year#X, volume#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [partial_sum(volume#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] -(124) Exchange +(134) Exchange Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(supp_nation#X, cust_nation#X, l_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(125) HashAggregate +(135) HashAggregate Input [5]: [supp_nation#X, cust_nation#X, l_year#X, sum#X, isEmpty#X] Keys [3]: [supp_nation#X, cust_nation#X, l_year#X] Functions [1]: [sum(volume#X)] Aggregate Attributes [1]: [sum(volume#X)#X] Results [4]: [supp_nation#X, cust_nation#X, l_year#X, sum(volume#X)#X AS revenue#X] -(126) Exchange +(136) Exchange Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: rangepartitioning(supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(127) Sort +(137) Sort Input [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: [supp_nation#X ASC NULLS FIRST, cust_nation#X ASC NULLS FIRST, l_year#X ASC NULLS FIRST], true, 0 -(128) AdaptiveSparkPlan +(138) AdaptiveSparkPlan Output [4]: [supp_nation#X, cust_nation#X, l_year#X, revenue#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt index cfc03953fe18a..5427956bb9d1a 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/8.txt @@ -1,150 +1,165 @@ == Physical Plan == -AdaptiveSparkPlan (177) +AdaptiveSparkPlan (192) +- == Final Plan == - VeloxColumnarToRowExec (125) - +- ^ SortExecTransformer (123) - +- ^ InputIteratorTransformer (122) - +- ShuffleQueryStage (120), Statistics(X) - +- ColumnarExchange (119) - +- ^ ProjectExecTransformer (117) - +- ^ RegularHashAggregateExecTransformer (116) - +- ^ InputIteratorTransformer (115) - +- ShuffleQueryStage (113), Statistics(X) - +- ColumnarExchange (112) - +- ^ ProjectExecTransformer (110) - +- ^ FlushableHashAggregateExecTransformer (109) - +- ^ ProjectExecTransformer (108) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (107) - :- ^ InputIteratorTransformer (98) - : +- ShuffleQueryStage (96), Statistics(X) - : +- ColumnarExchange (95) - : +- ^ ProjectExecTransformer (93) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (92) - : :- ^ InputIteratorTransformer (83) - : : +- ShuffleQueryStage (81), Statistics(X) - : : +- ColumnarExchange (80) - : : +- ^ ProjectExecTransformer (78) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - : : :- ^ InputIteratorTransformer (68) - : : : +- ShuffleQueryStage (66), Statistics(X) - : : : +- ColumnarExchange (65) - : : : +- ^ ProjectExecTransformer (63) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : : : :- ^ InputIteratorTransformer (53) - : : : : +- ShuffleQueryStage (51), Statistics(X) - : : : : +- ColumnarExchange (50) - : : : : +- ^ ProjectExecTransformer (48) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : : : :- ^ InputIteratorTransformer (38) - : : : : : +- ShuffleQueryStage (36), Statistics(X) - : : : : : +- ColumnarExchange (35) - : : : : : +- ^ ProjectExecTransformer (33) - : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : : : :- ^ InputIteratorTransformer (23) - : : : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : : : +- ColumnarExchange (20) - : : : : : : +- ^ ProjectExecTransformer (18) - : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : : : :- ^ InputIteratorTransformer (8) - : : : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : : : +- ColumnarExchange (5) - : : : : : : : +- ^ ProjectExecTransformer (3) - : : : : : : : +- ^ FilterExecTransformer (2) - : : : : : : : +- ^ Scan parquet (1) - : : : : : : +- ^ InputIteratorTransformer (16) - : : : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : : : +- ColumnarExchange (13) - : : : : : : +- ^ ProjectExecTransformer (11) - : : : : : : +- ^ FilterExecTransformer (10) - : : : : : : +- ^ Scan parquet (9) - : : : : : +- ^ InputIteratorTransformer (31) - : : : : : +- ShuffleQueryStage (29), Statistics(X) - : : : : : +- ColumnarExchange (28) - : : : : : +- ^ ProjectExecTransformer (26) - : : : : : +- ^ FilterExecTransformer (25) - : : : : : +- ^ Scan parquet (24) - : : : : +- ^ InputIteratorTransformer (46) - : : : : +- ShuffleQueryStage (44), Statistics(X) - : : : : +- ColumnarExchange (43) - : : : : +- ^ ProjectExecTransformer (41) - : : : : +- ^ FilterExecTransformer (40) - : : : : +- ^ Scan parquet (39) - : : : +- ^ InputIteratorTransformer (61) - : : : +- ShuffleQueryStage (59), Statistics(X) - : : : +- ColumnarExchange (58) - : : : +- ^ ProjectExecTransformer (56) - : : : +- ^ FilterExecTransformer (55) - : : : +- ^ Scan parquet (54) - : : +- ^ InputIteratorTransformer (76) - : : +- ShuffleQueryStage (74), Statistics(X) - : : +- ColumnarExchange (73) - : : +- ^ ProjectExecTransformer (71) - : : +- ^ FilterExecTransformer (70) - : : +- ^ Scan parquet (69) - : +- ^ InputIteratorTransformer (91) - : +- ShuffleQueryStage (89), Statistics(X) - : +- ColumnarExchange (88) - : +- ^ ProjectExecTransformer (86) - : +- ^ FilterExecTransformer (85) - : +- ^ Scan parquet (84) - +- ^ InputIteratorTransformer (106) - +- ShuffleQueryStage (104), Statistics(X) - +- ColumnarExchange (103) - +- ^ ProjectExecTransformer (101) - +- ^ FilterExecTransformer (100) - +- ^ Scan parquet (99) + VeloxColumnarToRowExec (140) + +- ^ SortExecTransformer (138) + +- ^ InputIteratorTransformer (137) + +- ShuffleQueryStage (135), Statistics(X) + +- ColumnarExchange (134) + +- VeloxAppendBatches (133) + +- ^ ProjectExecTransformer (131) + +- ^ RegularHashAggregateExecTransformer (130) + +- ^ InputIteratorTransformer (129) + +- ShuffleQueryStage (127), Statistics(X) + +- ColumnarExchange (126) + +- ^ ProjectExecTransformer (124) + +- ^ FlushableHashAggregateExecTransformer (123) + +- ^ ProjectExecTransformer (122) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (121) + :- ^ InputIteratorTransformer (111) + : +- ShuffleQueryStage (109), Statistics(X) + : +- ColumnarExchange (108) + : +- VeloxAppendBatches (107) + : +- ^ ProjectExecTransformer (105) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (104) + : :- ^ InputIteratorTransformer (94) + : : +- ShuffleQueryStage (92), Statistics(X) + : : +- ColumnarExchange (91) + : : +- VeloxAppendBatches (90) + : : +- ^ ProjectExecTransformer (88) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + : : :- ^ InputIteratorTransformer (77) + : : : +- ShuffleQueryStage (75), Statistics(X) + : : : +- ColumnarExchange (74) + : : : +- VeloxAppendBatches (73) + : : : +- ^ ProjectExecTransformer (71) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : : : :- ^ InputIteratorTransformer (60) + : : : : +- ShuffleQueryStage (58), Statistics(X) + : : : : +- ColumnarExchange (57) + : : : : +- VeloxAppendBatches (56) + : : : : +- ^ ProjectExecTransformer (54) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : : : :- ^ InputIteratorTransformer (43) + : : : : : +- ShuffleQueryStage (41), Statistics(X) + : : : : : +- ColumnarExchange (40) + : : : : : +- VeloxAppendBatches (39) + : : : : : +- ^ ProjectExecTransformer (37) + : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : : : :- ^ InputIteratorTransformer (26) + : : : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : : : +- ColumnarExchange (23) + : : : : : : +- VeloxAppendBatches (22) + : : : : : : +- ^ ProjectExecTransformer (20) + : : : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : : : :- ^ InputIteratorTransformer (9) + : : : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : : : +- ColumnarExchange (6) + : : : : : : : +- VeloxAppendBatches (5) + : : : : : : : +- ^ ProjectExecTransformer (3) + : : : : : : : +- ^ FilterExecTransformer (2) + : : : : : : : +- ^ Scan parquet (1) + : : : : : : +- ^ InputIteratorTransformer (18) + : : : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : : : +- ColumnarExchange (15) + : : : : : : +- VeloxAppendBatches (14) + : : : : : : +- ^ ProjectExecTransformer (12) + : : : : : : +- ^ FilterExecTransformer (11) + : : : : : : +- ^ Scan parquet (10) + : : : : : +- ^ InputIteratorTransformer (35) + : : : : : +- ShuffleQueryStage (33), Statistics(X) + : : : : : +- ColumnarExchange (32) + : : : : : +- VeloxAppendBatches (31) + : : : : : +- ^ ProjectExecTransformer (29) + : : : : : +- ^ FilterExecTransformer (28) + : : : : : +- ^ Scan parquet (27) + : : : : +- ^ InputIteratorTransformer (52) + : : : : +- ShuffleQueryStage (50), Statistics(X) + : : : : +- ColumnarExchange (49) + : : : : +- VeloxAppendBatches (48) + : : : : +- ^ ProjectExecTransformer (46) + : : : : +- ^ FilterExecTransformer (45) + : : : : +- ^ Scan parquet (44) + : : : +- ^ InputIteratorTransformer (69) + : : : +- ShuffleQueryStage (67), Statistics(X) + : : : +- ColumnarExchange (66) + : : : +- VeloxAppendBatches (65) + : : : +- ^ ProjectExecTransformer (63) + : : : +- ^ FilterExecTransformer (62) + : : : +- ^ Scan parquet (61) + : : +- ^ InputIteratorTransformer (86) + : : +- ShuffleQueryStage (84), Statistics(X) + : : +- ColumnarExchange (83) + : : +- VeloxAppendBatches (82) + : : +- ^ ProjectExecTransformer (80) + : : +- ^ FilterExecTransformer (79) + : : +- ^ Scan parquet (78) + : +- ^ InputIteratorTransformer (103) + : +- ShuffleQueryStage (101), Statistics(X) + : +- ColumnarExchange (100) + : +- VeloxAppendBatches (99) + : +- ^ ProjectExecTransformer (97) + : +- ^ FilterExecTransformer (96) + : +- ^ Scan parquet (95) + +- ^ InputIteratorTransformer (120) + +- ShuffleQueryStage (118), Statistics(X) + +- ColumnarExchange (117) + +- VeloxAppendBatches (116) + +- ^ ProjectExecTransformer (114) + +- ^ FilterExecTransformer (113) + +- ^ Scan parquet (112) +- == Initial Plan == - Sort (176) - +- Exchange (175) - +- HashAggregate (174) - +- Exchange (173) - +- HashAggregate (172) - +- Project (171) - +- ShuffledHashJoin Inner BuildRight (170) - :- Exchange (165) - : +- Project (164) - : +- ShuffledHashJoin Inner BuildRight (163) - : :- Exchange (159) - : : +- Project (158) - : : +- ShuffledHashJoin Inner BuildRight (157) - : : :- Exchange (153) - : : : +- Project (152) - : : : +- ShuffledHashJoin Inner BuildRight (151) - : : : :- Exchange (147) - : : : : +- Project (146) - : : : : +- ShuffledHashJoin Inner BuildRight (145) - : : : : :- Exchange (141) - : : : : : +- Project (140) - : : : : : +- ShuffledHashJoin Inner BuildRight (139) - : : : : : :- Exchange (135) - : : : : : : +- Project (134) - : : : : : : +- ShuffledHashJoin Inner BuildLeft (133) - : : : : : : :- Exchange (129) - : : : : : : : +- Project (128) - : : : : : : : +- Filter (127) - : : : : : : : +- Scan parquet (126) - : : : : : : +- Exchange (132) - : : : : : : +- Filter (131) - : : : : : : +- Scan parquet (130) - : : : : : +- Exchange (138) - : : : : : +- Filter (137) - : : : : : +- Scan parquet (136) - : : : : +- Exchange (144) - : : : : +- Filter (143) - : : : : +- Scan parquet (142) - : : : +- Exchange (150) - : : : +- Filter (149) - : : : +- Scan parquet (148) - : : +- Exchange (156) - : : +- Filter (155) - : : +- Scan parquet (154) - : +- Exchange (162) - : +- Filter (161) - : +- Scan parquet (160) - +- Exchange (169) - +- Project (168) - +- Filter (167) - +- Scan parquet (166) + Sort (191) + +- Exchange (190) + +- HashAggregate (189) + +- Exchange (188) + +- HashAggregate (187) + +- Project (186) + +- ShuffledHashJoin Inner BuildRight (185) + :- Exchange (180) + : +- Project (179) + : +- ShuffledHashJoin Inner BuildRight (178) + : :- Exchange (174) + : : +- Project (173) + : : +- ShuffledHashJoin Inner BuildRight (172) + : : :- Exchange (168) + : : : +- Project (167) + : : : +- ShuffledHashJoin Inner BuildRight (166) + : : : :- Exchange (162) + : : : : +- Project (161) + : : : : +- ShuffledHashJoin Inner BuildRight (160) + : : : : :- Exchange (156) + : : : : : +- Project (155) + : : : : : +- ShuffledHashJoin Inner BuildRight (154) + : : : : : :- Exchange (150) + : : : : : : +- Project (149) + : : : : : : +- ShuffledHashJoin Inner BuildLeft (148) + : : : : : : :- Exchange (144) + : : : : : : : +- Project (143) + : : : : : : : +- Filter (142) + : : : : : : : +- Scan parquet (141) + : : : : : : +- Exchange (147) + : : : : : : +- Filter (146) + : : : : : : +- Scan parquet (145) + : : : : : +- Exchange (153) + : : : : : +- Filter (152) + : : : : : +- Scan parquet (151) + : : : : +- Exchange (159) + : : : : +- Filter (158) + : : : : +- Scan parquet (157) + : : : +- Exchange (165) + : : : +- Filter (164) + : : : +- Scan parquet (163) + : : +- Exchange (171) + : : +- Filter (170) + : : +- Scan parquet (169) + : +- Exchange (177) + : +- Filter (176) + : +- Scan parquet (175) + +- Exchange (184) + +- Project (183) + +- Filter (182) + +- Scan parquet (181) (1) Scan parquet @@ -166,746 +181,806 @@ Input [2]: [p_partkey#X, p_type#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [6]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [6]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [5]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [5]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [5]: [hash_partition_key#X, l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [6]: [hash(o_custkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] +Arguments: X + +(57) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [c_custkey#X, c_nationkey#X] Arguments: (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(c_custkey#X, 42) AS hash_partition_key#X, c_custkey#X, c_nationkey#X] Input [2]: [c_custkey#X, c_nationkey#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [c_custkey#X, c_nationkey#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [c_custkey#X, c_nationkey#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [c_custkey#X, c_nationkey#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [c_custkey#X, c_nationkey#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [6]: [hash(c_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] +Arguments: X + +(74) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_regionkey#X] Input [2]: [n_nationkey#X, n_regionkey#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_regionkey#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_regionkey#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_regionkey#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_regionkey#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [6]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(79) WholeStageCodegenTransformer (X) +(89) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: false -(80) ColumnarExchange +(90) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] +Arguments: X + +(91) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X], [plan_id=X], [id=#X] -(81) ShuffleQueryStage +(92) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: X -(82) InputAdapter +(93) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(83) InputIteratorTransformer +(94) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] -(84) Scan parquet +(95) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(85) FilterExecTransformer +(96) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(86) ProjectExecTransformer +(97) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(87) WholeStageCodegenTransformer (X) +(98) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(88) ColumnarExchange +(99) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(100) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(101) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(90) InputAdapter +(102) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(91) InputIteratorTransformer +(103) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(92) ShuffledHashJoinExecTransformer +(104) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(93) ProjectExecTransformer +(105) ProjectExecTransformer Output [6]: [hash(n_regionkey#X, 42) AS hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(94) WholeStageCodegenTransformer (X) +(106) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: false -(95) ColumnarExchange +(107) VeloxAppendBatches +Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] +Arguments: X + +(108) ColumnarExchange Input [6]: [hash_partition_key#X, l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X], [plan_id=X], [id=#X] -(96) ShuffleQueryStage +(109) ShuffleQueryStage Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: X -(97) InputAdapter +(110) InputAdapter Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(98) InputIteratorTransformer +(111) InputIteratorTransformer Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] -(99) Scan parquet +(112) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(100) FilterExecTransformer +(113) FilterExecTransformer Input [2]: [r_regionkey#X, r_name#X] Arguments: ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(101) ProjectExecTransformer +(114) ProjectExecTransformer Output [2]: [hash(r_regionkey#X, 42) AS hash_partition_key#X, r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(102) WholeStageCodegenTransformer (X) +(115) WholeStageCodegenTransformer (X) Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: false -(103) ColumnarExchange +(116) VeloxAppendBatches +Input [2]: [hash_partition_key#X, r_regionkey#X] +Arguments: X + +(117) ColumnarExchange Input [2]: [hash_partition_key#X, r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [r_regionkey#X], [plan_id=X], [id=#X] -(104) ShuffleQueryStage +(118) ShuffleQueryStage Output [1]: [r_regionkey#X] Arguments: X -(105) InputAdapter +(119) InputAdapter Input [1]: [r_regionkey#X] -(106) InputIteratorTransformer +(120) InputIteratorTransformer Input [1]: [r_regionkey#X] -(107) ShuffledHashJoinExecTransformer +(121) ShuffledHashJoinExecTransformer Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(108) ProjectExecTransformer +(122) ProjectExecTransformer Output [4]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X, CASE WHEN (n_name#X = BRAZIL) THEN (l_extendedprice#X * (1 - l_discount#X)) ELSE 0.0000 END AS _pre_X#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(109) FlushableHashAggregateExecTransformer +(123) FlushableHashAggregateExecTransformer Input [4]: [o_year#X, volume#X, nation#X, _pre_X#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(_pre_X#X), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(110) ProjectExecTransformer +(124) ProjectExecTransformer Output [6]: [hash(o_year#X, 42) AS hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(111) WholeStageCodegenTransformer (X) +(125) WholeStageCodegenTransformer (X) Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: false -(112) ColumnarExchange +(126) ColumnarExchange Input [6]: [hash_partition_key#X, o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(113) ShuffleQueryStage +(127) ShuffleQueryStage Output [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: X -(114) InputAdapter +(128) InputAdapter Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(115) InputIteratorTransformer +(129) InputIteratorTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(116) RegularHashAggregateExecTransformer +(130) RegularHashAggregateExecTransformer Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(117) ProjectExecTransformer +(131) ProjectExecTransformer Output [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] Input [3]: [o_year#X, sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] -(118) WholeStageCodegenTransformer (X) +(132) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(119) ColumnarExchange +(133) VeloxAppendBatches +Input [2]: [o_year#X, mkt_share#X] +Arguments: X + +(134) ColumnarExchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(120) ShuffleQueryStage +(135) ShuffleQueryStage Output [2]: [o_year#X, mkt_share#X] Arguments: X -(121) InputAdapter +(136) InputAdapter Input [2]: [o_year#X, mkt_share#X] -(122) InputIteratorTransformer +(137) InputIteratorTransformer Input [2]: [o_year#X, mkt_share#X] -(123) SortExecTransformer +(138) SortExecTransformer Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(124) WholeStageCodegenTransformer (X) +(139) WholeStageCodegenTransformer (X) Input [2]: [o_year#X, mkt_share#X] Arguments: false -(125) VeloxColumnarToRowExec +(140) VeloxColumnarToRowExec Input [2]: [o_year#X, mkt_share#X] -(126) Scan parquet +(141) Scan parquet Output [2]: [p_partkey#X, p_type#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_type), EqualTo(p_type,ECONOMY ANODIZED STEEL), IsNotNull(p_partkey)] ReadSchema: struct -(127) Filter +(142) Filter Input [2]: [p_partkey#X, p_type#X] Condition : ((isnotnull(p_type#X) AND (p_type#X = ECONOMY ANODIZED STEEL)) AND isnotnull(p_partkey#X)) -(128) Project +(143) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_type#X] -(129) Exchange +(144) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) Scan parquet +(145) Scan parquet Output [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(131) Filter +(146) Filter Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(132) Exchange +(147) Exchange Input [5]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(133) ShuffledHashJoin +(148) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(134) Project +(149) Project Output [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Input [6]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] -(135) Exchange +(150) Exchange Input [4]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(136) Scan parquet +(151) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(137) Filter +(152) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(138) Exchange +(153) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(139) ShuffledHashJoin +(154) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(140) Project +(155) Project Output [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [6]: [l_orderkey#X, l_suppkey#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(141) Exchange +(156) Exchange Input [4]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(142) Scan parquet +(157) Scan parquet Output [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderdate), GreaterThanOrEqual(o_orderdate,1995-01-01), LessThanOrEqual(o_orderdate,1996-12-31), IsNotNull(o_orderkey), IsNotNull(o_custkey)] ReadSchema: struct -(143) Filter +(158) Filter Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Condition : ((((isnotnull(o_orderdate#X) AND (o_orderdate#X >= 1995-01-01)) AND (o_orderdate#X <= 1996-12-31)) AND isnotnull(o_orderkey#X)) AND isnotnull(o_custkey#X)) -(144) Exchange +(159) Exchange Input [3]: [o_orderkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(145) ShuffledHashJoin +(160) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(146) Project +(161) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Input [7]: [l_orderkey#X, l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderkey#X, o_custkey#X, o_orderdate#X] -(147) Exchange +(162) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X] Arguments: hashpartitioning(o_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(148) Scan parquet +(163) Scan parquet Output [2]: [c_custkey#X, c_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(c_custkey), IsNotNull(c_nationkey)] ReadSchema: struct -(149) Filter +(164) Filter Input [2]: [c_custkey#X, c_nationkey#X] Condition : (isnotnull(c_custkey#X) AND isnotnull(c_nationkey#X)) -(150) Exchange +(165) Exchange Input [2]: [c_custkey#X, c_nationkey#X] Arguments: hashpartitioning(c_custkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(151) ShuffledHashJoin +(166) ShuffledHashJoin Left keys [1]: [o_custkey#X] Right keys [1]: [c_custkey#X] Join type: Inner Join condition: None -(152) Project +(167) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_custkey#X, o_orderdate#X, c_custkey#X, c_nationkey#X] -(153) Exchange +(168) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X] Arguments: hashpartitioning(c_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(154) Scan parquet +(169) Scan parquet Output [2]: [n_nationkey#X, n_regionkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey), IsNotNull(n_regionkey)] ReadSchema: struct -(155) Filter +(170) Filter Input [2]: [n_nationkey#X, n_regionkey#X] Condition : (isnotnull(n_nationkey#X) AND isnotnull(n_regionkey#X)) -(156) Exchange +(171) Exchange Input [2]: [n_nationkey#X, n_regionkey#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(157) ShuffledHashJoin +(172) ShuffledHashJoin Left keys [1]: [c_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(158) Project +(173) Project Output [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, c_nationkey#X, n_nationkey#X, n_regionkey#X] -(159) Exchange +(174) Exchange Input [5]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(160) Scan parquet +(175) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(161) Filter +(176) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(162) Exchange +(177) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(163) ShuffledHashJoin +(178) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(164) Project +(179) Project Output [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Input [7]: [l_extendedprice#X, l_discount#X, s_nationkey#X, o_orderdate#X, n_regionkey#X, n_nationkey#X, n_name#X] -(165) Exchange +(180) Exchange Input [5]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X] Arguments: hashpartitioning(n_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(166) Scan parquet +(181) Scan parquet Output [2]: [r_regionkey#X, r_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(r_name), EqualTo(r_name,AMERICA), IsNotNull(r_regionkey)] ReadSchema: struct -(167) Filter +(182) Filter Input [2]: [r_regionkey#X, r_name#X] Condition : ((isnotnull(r_name#X) AND (r_name#X = AMERICA)) AND isnotnull(r_regionkey#X)) -(168) Project +(183) Project Output [1]: [r_regionkey#X] Input [2]: [r_regionkey#X, r_name#X] -(169) Exchange +(184) Exchange Input [1]: [r_regionkey#X] Arguments: hashpartitioning(r_regionkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(170) ShuffledHashJoin +(185) ShuffledHashJoin Left keys [1]: [n_regionkey#X] Right keys [1]: [r_regionkey#X] Join type: Inner Join condition: None -(171) Project +(186) Project Output [3]: [year(o_orderdate#X) AS o_year#X, (l_extendedprice#X * (1 - l_discount#X)) AS volume#X, n_name#X AS nation#X] Input [6]: [l_extendedprice#X, l_discount#X, o_orderdate#X, n_regionkey#X, n_name#X, r_regionkey#X] -(172) HashAggregate +(187) HashAggregate Input [3]: [o_year#X, volume#X, nation#X] Keys [1]: [o_year#X] Functions [2]: [partial_sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), partial_sum(volume#X)] Aggregate Attributes [4]: [sum#X, isEmpty#X, sum#X, isEmpty#X] Results [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] -(173) Exchange +(188) Exchange Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Arguments: hashpartitioning(o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(174) HashAggregate +(189) HashAggregate Input [5]: [o_year#X, sum#X, isEmpty#X, sum#X, isEmpty#X] Keys [1]: [o_year#X] Functions [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END), sum(volume#X)] Aggregate Attributes [2]: [sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X, sum(volume#X)#X] Results [2]: [o_year#X, (sum(CASE WHEN (nation#X = BRAZIL) THEN volume#X ELSE 0.0000 END)#X / sum(volume#X)#X) AS mkt_share#X] -(175) Exchange +(190) Exchange Input [2]: [o_year#X, mkt_share#X] Arguments: rangepartitioning(o_year#X ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(176) Sort +(191) Sort Input [2]: [o_year#X, mkt_share#X] Arguments: [o_year#X ASC NULLS FIRST], true, 0 -(177) AdaptiveSparkPlan +(192) AdaptiveSparkPlan Output [2]: [o_year#X, mkt_share#X] Arguments: isFinalPlan=true \ No newline at end of file diff --git a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt index 9a8c9a87aac2a..af275ac9926ce 100644 --- a/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt +++ b/backends-velox/src/test/resources/tpch-approved-plan/v1/spark34/9.txt @@ -1,114 +1,125 @@ == Physical Plan == -AdaptiveSparkPlan (133) +AdaptiveSparkPlan (144) +- == Final Plan == - VeloxColumnarToRowExec (94) - +- ^ SortExecTransformer (92) - +- ^ InputIteratorTransformer (91) - +- ShuffleQueryStage (89), Statistics(X) - +- ColumnarExchange (88) - +- ^ RegularHashAggregateExecTransformer (86) - +- ^ InputIteratorTransformer (85) - +- ShuffleQueryStage (83), Statistics(X) - +- ColumnarExchange (82) - +- ^ ProjectExecTransformer (80) - +- ^ FlushableHashAggregateExecTransformer (79) - +- ^ ProjectExecTransformer (78) - +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (77) - :- ^ InputIteratorTransformer (68) - : +- ShuffleQueryStage (66), Statistics(X) - : +- ColumnarExchange (65) - : +- ^ ProjectExecTransformer (63) - : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (62) - : :- ^ InputIteratorTransformer (53) - : : +- ShuffleQueryStage (51), Statistics(X) - : : +- ColumnarExchange (50) - : : +- ^ ProjectExecTransformer (48) - : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (47) - : : :- ^ InputIteratorTransformer (38) - : : : +- ShuffleQueryStage (36), Statistics(X) - : : : +- ColumnarExchange (35) - : : : +- ^ ProjectExecTransformer (33) - : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (32) - : : : :- ^ InputIteratorTransformer (23) - : : : : +- ShuffleQueryStage (21), Statistics(X) - : : : : +- ColumnarExchange (20) - : : : : +- ^ ProjectExecTransformer (18) - : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (17) - : : : : :- ^ InputIteratorTransformer (8) - : : : : : +- ShuffleQueryStage (6), Statistics(X) - : : : : : +- ColumnarExchange (5) - : : : : : +- ^ ProjectExecTransformer (3) - : : : : : +- ^ FilterExecTransformer (2) - : : : : : +- ^ Scan parquet (1) - : : : : +- ^ InputIteratorTransformer (16) - : : : : +- ShuffleQueryStage (14), Statistics(X) - : : : : +- ColumnarExchange (13) - : : : : +- ^ ProjectExecTransformer (11) - : : : : +- ^ FilterExecTransformer (10) - : : : : +- ^ Scan parquet (9) - : : : +- ^ InputIteratorTransformer (31) - : : : +- ShuffleQueryStage (29), Statistics(X) - : : : +- ColumnarExchange (28) - : : : +- ^ ProjectExecTransformer (26) - : : : +- ^ FilterExecTransformer (25) - : : : +- ^ Scan parquet (24) - : : +- ^ InputIteratorTransformer (46) - : : +- ShuffleQueryStage (44), Statistics(X) - : : +- ColumnarExchange (43) - : : +- ^ ProjectExecTransformer (41) - : : +- ^ FilterExecTransformer (40) - : : +- ^ Scan parquet (39) - : +- ^ InputIteratorTransformer (61) - : +- ShuffleQueryStage (59), Statistics(X) - : +- ColumnarExchange (58) - : +- ^ ProjectExecTransformer (56) - : +- ^ FilterExecTransformer (55) - : +- ^ Scan parquet (54) - +- ^ InputIteratorTransformer (76) - +- ShuffleQueryStage (74), Statistics(X) - +- ColumnarExchange (73) - +- ^ ProjectExecTransformer (71) - +- ^ FilterExecTransformer (70) - +- ^ Scan parquet (69) + VeloxColumnarToRowExec (105) + +- ^ SortExecTransformer (103) + +- ^ InputIteratorTransformer (102) + +- ShuffleQueryStage (100), Statistics(X) + +- ColumnarExchange (99) + +- VeloxAppendBatches (98) + +- ^ RegularHashAggregateExecTransformer (96) + +- ^ InputIteratorTransformer (95) + +- ShuffleQueryStage (93), Statistics(X) + +- ColumnarExchange (92) + +- ^ ProjectExecTransformer (90) + +- ^ FlushableHashAggregateExecTransformer (89) + +- ^ ProjectExecTransformer (88) + +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (87) + :- ^ InputIteratorTransformer (77) + : +- ShuffleQueryStage (75), Statistics(X) + : +- ColumnarExchange (74) + : +- VeloxAppendBatches (73) + : +- ^ ProjectExecTransformer (71) + : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (70) + : :- ^ InputIteratorTransformer (60) + : : +- ShuffleQueryStage (58), Statistics(X) + : : +- ColumnarExchange (57) + : : +- VeloxAppendBatches (56) + : : +- ^ ProjectExecTransformer (54) + : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (53) + : : :- ^ InputIteratorTransformer (43) + : : : +- ShuffleQueryStage (41), Statistics(X) + : : : +- ColumnarExchange (40) + : : : +- VeloxAppendBatches (39) + : : : +- ^ ProjectExecTransformer (37) + : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildRight (36) + : : : :- ^ InputIteratorTransformer (26) + : : : : +- ShuffleQueryStage (24), Statistics(X) + : : : : +- ColumnarExchange (23) + : : : : +- VeloxAppendBatches (22) + : : : : +- ^ ProjectExecTransformer (20) + : : : : +- ^ ShuffledHashJoinExecTransformer Inner BuildLeft (19) + : : : : :- ^ InputIteratorTransformer (9) + : : : : : +- ShuffleQueryStage (7), Statistics(X) + : : : : : +- ColumnarExchange (6) + : : : : : +- VeloxAppendBatches (5) + : : : : : +- ^ ProjectExecTransformer (3) + : : : : : +- ^ FilterExecTransformer (2) + : : : : : +- ^ Scan parquet (1) + : : : : +- ^ InputIteratorTransformer (18) + : : : : +- ShuffleQueryStage (16), Statistics(X) + : : : : +- ColumnarExchange (15) + : : : : +- VeloxAppendBatches (14) + : : : : +- ^ ProjectExecTransformer (12) + : : : : +- ^ FilterExecTransformer (11) + : : : : +- ^ Scan parquet (10) + : : : +- ^ InputIteratorTransformer (35) + : : : +- ShuffleQueryStage (33), Statistics(X) + : : : +- ColumnarExchange (32) + : : : +- VeloxAppendBatches (31) + : : : +- ^ ProjectExecTransformer (29) + : : : +- ^ FilterExecTransformer (28) + : : : +- ^ Scan parquet (27) + : : +- ^ InputIteratorTransformer (52) + : : +- ShuffleQueryStage (50), Statistics(X) + : : +- ColumnarExchange (49) + : : +- VeloxAppendBatches (48) + : : +- ^ ProjectExecTransformer (46) + : : +- ^ FilterExecTransformer (45) + : : +- ^ Scan parquet (44) + : +- ^ InputIteratorTransformer (69) + : +- ShuffleQueryStage (67), Statistics(X) + : +- ColumnarExchange (66) + : +- VeloxAppendBatches (65) + : +- ^ ProjectExecTransformer (63) + : +- ^ FilterExecTransformer (62) + : +- ^ Scan parquet (61) + +- ^ InputIteratorTransformer (86) + +- ShuffleQueryStage (84), Statistics(X) + +- ColumnarExchange (83) + +- VeloxAppendBatches (82) + +- ^ ProjectExecTransformer (80) + +- ^ FilterExecTransformer (79) + +- ^ Scan parquet (78) +- == Initial Plan == - Sort (132) - +- Exchange (131) - +- HashAggregate (130) - +- Exchange (129) - +- HashAggregate (128) - +- Project (127) - +- ShuffledHashJoin Inner BuildRight (126) - :- Exchange (122) - : +- Project (121) - : +- ShuffledHashJoin Inner BuildRight (120) - : :- Exchange (116) - : : +- Project (115) - : : +- ShuffledHashJoin Inner BuildRight (114) - : : :- Exchange (110) - : : : +- Project (109) - : : : +- ShuffledHashJoin Inner BuildRight (108) - : : : :- Exchange (104) - : : : : +- Project (103) - : : : : +- ShuffledHashJoin Inner BuildLeft (102) - : : : : :- Exchange (98) - : : : : : +- Project (97) - : : : : : +- Filter (96) - : : : : : +- Scan parquet (95) - : : : : +- Exchange (101) - : : : : +- Filter (100) - : : : : +- Scan parquet (99) - : : : +- Exchange (107) - : : : +- Filter (106) - : : : +- Scan parquet (105) - : : +- Exchange (113) - : : +- Filter (112) - : : +- Scan parquet (111) - : +- Exchange (119) - : +- Filter (118) - : +- Scan parquet (117) - +- Exchange (125) - +- Filter (124) - +- Scan parquet (123) + Sort (143) + +- Exchange (142) + +- HashAggregate (141) + +- Exchange (140) + +- HashAggregate (139) + +- Project (138) + +- ShuffledHashJoin Inner BuildRight (137) + :- Exchange (133) + : +- Project (132) + : +- ShuffledHashJoin Inner BuildRight (131) + : :- Exchange (127) + : : +- Project (126) + : : +- ShuffledHashJoin Inner BuildRight (125) + : : :- Exchange (121) + : : : +- Project (120) + : : : +- ShuffledHashJoin Inner BuildRight (119) + : : : :- Exchange (115) + : : : : +- Project (114) + : : : : +- ShuffledHashJoin Inner BuildLeft (113) + : : : : :- Exchange (109) + : : : : : +- Project (108) + : : : : : +- Filter (107) + : : : : : +- Scan parquet (106) + : : : : +- Exchange (112) + : : : : +- Filter (111) + : : : : +- Scan parquet (110) + : : : +- Exchange (118) + : : : +- Filter (117) + : : : +- Scan parquet (116) + : : +- Exchange (124) + : : +- Filter (123) + : : +- Scan parquet (122) + : +- Exchange (130) + : +- Filter (129) + : +- Scan parquet (128) + +- Exchange (136) + +- Filter (135) + +- Scan parquet (134) (1) Scan parquet @@ -130,558 +141,602 @@ Input [2]: [p_partkey#X, p_name#X] Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: false -(5) ColumnarExchange +(5) VeloxAppendBatches +Input [2]: [hash_partition_key#X, p_partkey#X] +Arguments: X + +(6) ColumnarExchange Input [2]: [hash_partition_key#X, p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [p_partkey#X], [plan_id=X], [id=#X] -(6) ShuffleQueryStage +(7) ShuffleQueryStage Output [1]: [p_partkey#X] Arguments: X -(7) InputAdapter +(8) InputAdapter Input [1]: [p_partkey#X] -(8) InputIteratorTransformer +(9) InputIteratorTransformer Input [1]: [p_partkey#X] -(9) Scan parquet +(10) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(10) FilterExecTransformer +(11) FilterExecTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(11) ProjectExecTransformer +(12) ProjectExecTransformer Output [7]: [hash(l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(12) WholeStageCodegenTransformer (X) +(13) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(13) ColumnarExchange +(14) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(15) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(14) ShuffleQueryStage +(16) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(15) InputAdapter +(17) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(16) InputIteratorTransformer +(18) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(17) ShuffledHashJoinExecTransformer +(19) ShuffledHashJoinExecTransformer Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(18) ProjectExecTransformer +(20) ProjectExecTransformer Output [7]: [hash(l_suppkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(19) WholeStageCodegenTransformer (X) +(21) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: false -(20) ColumnarExchange +(22) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] +Arguments: X + +(23) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X], [plan_id=X], [id=#X] -(21) ShuffleQueryStage +(24) ShuffleQueryStage Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: X -(22) InputAdapter +(25) InputAdapter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(23) InputIteratorTransformer +(26) InputIteratorTransformer Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(24) Scan parquet +(27) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(25) FilterExecTransformer +(28) FilterExecTransformer Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(26) ProjectExecTransformer +(29) ProjectExecTransformer Output [3]: [hash(s_suppkey#X, 42) AS hash_partition_key#X, s_suppkey#X, s_nationkey#X] Input [2]: [s_suppkey#X, s_nationkey#X] -(27) WholeStageCodegenTransformer (X) +(30) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: false -(28) ColumnarExchange +(31) VeloxAppendBatches +Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] +Arguments: X + +(32) ColumnarExchange Input [3]: [hash_partition_key#X, s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [s_suppkey#X, s_nationkey#X], [plan_id=X], [id=#X] -(29) ShuffleQueryStage +(33) ShuffleQueryStage Output [2]: [s_suppkey#X, s_nationkey#X] Arguments: X -(30) InputAdapter +(34) InputAdapter Input [2]: [s_suppkey#X, s_nationkey#X] -(31) InputIteratorTransformer +(35) InputIteratorTransformer Input [2]: [s_suppkey#X, s_nationkey#X] -(32) ShuffledHashJoinExecTransformer +(36) ShuffledHashJoinExecTransformer Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(33) ProjectExecTransformer +(37) ProjectExecTransformer Output [8]: [hash(l_suppkey#X, l_partkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(34) WholeStageCodegenTransformer (X) +(38) WholeStageCodegenTransformer (X) Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: false -(35) ColumnarExchange +(39) VeloxAppendBatches +Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] +Arguments: X + +(40) ColumnarExchange Input [8]: [hash_partition_key#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X], [plan_id=X], [id=#X] -(36) ShuffleQueryStage +(41) ShuffleQueryStage Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: X -(37) InputAdapter +(42) InputAdapter Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(38) InputIteratorTransformer +(43) InputIteratorTransformer Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] -(39) Scan parquet +(44) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(40) FilterExecTransformer +(45) FilterExecTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(41) ProjectExecTransformer +(46) ProjectExecTransformer Output [4]: [hash(ps_suppkey#X, ps_partkey#X, 42) AS hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(42) WholeStageCodegenTransformer (X) +(47) WholeStageCodegenTransformer (X) Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: false -(43) ColumnarExchange +(48) VeloxAppendBatches +Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] +Arguments: X + +(49) ColumnarExchange Input [4]: [hash_partition_key#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [ps_partkey#X, ps_suppkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(44) ShuffleQueryStage +(50) ShuffleQueryStage Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: X -(45) InputAdapter +(51) InputAdapter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(46) InputIteratorTransformer +(52) InputIteratorTransformer Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(47) ShuffledHashJoinExecTransformer +(53) ShuffledHashJoinExecTransformer Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(48) ProjectExecTransformer +(54) ProjectExecTransformer Output [7]: [hash(l_orderkey#X, 42) AS hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(49) WholeStageCodegenTransformer (X) +(55) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: false -(50) ColumnarExchange +(56) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] +Arguments: X + +(57) ColumnarExchange Input [7]: [hash_partition_key#X, l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X], [plan_id=X], [id=#X] -(51) ShuffleQueryStage +(58) ShuffleQueryStage Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: X -(52) InputAdapter +(59) InputAdapter Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(53) InputIteratorTransformer +(60) InputIteratorTransformer Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] -(54) Scan parquet +(61) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(55) FilterExecTransformer +(62) FilterExecTransformer Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: isnotnull(o_orderkey#X) -(56) ProjectExecTransformer +(63) ProjectExecTransformer Output [3]: [hash(o_orderkey#X, 42) AS hash_partition_key#X, o_orderkey#X, o_orderdate#X] Input [2]: [o_orderkey#X, o_orderdate#X] -(57) WholeStageCodegenTransformer (X) +(64) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: false -(58) ColumnarExchange +(65) VeloxAppendBatches +Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] +Arguments: X + +(66) ColumnarExchange Input [3]: [hash_partition_key#X, o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [o_orderkey#X, o_orderdate#X], [plan_id=X], [id=#X] -(59) ShuffleQueryStage +(67) ShuffleQueryStage Output [2]: [o_orderkey#X, o_orderdate#X] Arguments: X -(60) InputAdapter +(68) InputAdapter Input [2]: [o_orderkey#X, o_orderdate#X] -(61) InputIteratorTransformer +(69) InputIteratorTransformer Input [2]: [o_orderkey#X, o_orderdate#X] -(62) ShuffledHashJoinExecTransformer +(70) ShuffledHashJoinExecTransformer Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(63) ProjectExecTransformer +(71) ProjectExecTransformer Output [7]: [hash(s_nationkey#X, 42) AS hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(64) WholeStageCodegenTransformer (X) +(72) WholeStageCodegenTransformer (X) Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: false -(65) ColumnarExchange +(73) VeloxAppendBatches +Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] +Arguments: X + +(74) ColumnarExchange Input [7]: [hash_partition_key#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X], [plan_id=X], [id=#X] -(66) ShuffleQueryStage +(75) ShuffleQueryStage Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: X -(67) InputAdapter +(76) InputAdapter Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(68) InputIteratorTransformer +(77) InputIteratorTransformer Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] -(69) Scan parquet +(78) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(70) FilterExecTransformer +(79) FilterExecTransformer Input [2]: [n_nationkey#X, n_name#X] Arguments: isnotnull(n_nationkey#X) -(71) ProjectExecTransformer +(80) ProjectExecTransformer Output [3]: [hash(n_nationkey#X, 42) AS hash_partition_key#X, n_nationkey#X, n_name#X] Input [2]: [n_nationkey#X, n_name#X] -(72) WholeStageCodegenTransformer (X) +(81) WholeStageCodegenTransformer (X) Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: false -(73) ColumnarExchange +(82) VeloxAppendBatches +Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] +Arguments: X + +(83) ColumnarExchange Input [3]: [hash_partition_key#X, n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [n_nationkey#X, n_name#X], [plan_id=X], [id=#X] -(74) ShuffleQueryStage +(84) ShuffleQueryStage Output [2]: [n_nationkey#X, n_name#X] Arguments: X -(75) InputAdapter +(85) InputAdapter Input [2]: [n_nationkey#X, n_name#X] -(76) InputIteratorTransformer +(86) InputIteratorTransformer Input [2]: [n_nationkey#X, n_name#X] -(77) ShuffledHashJoinExecTransformer +(87) ShuffledHashJoinExecTransformer Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(78) ProjectExecTransformer +(88) ProjectExecTransformer Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(79) FlushableHashAggregateExecTransformer +(89) FlushableHashAggregateExecTransformer Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(80) ProjectExecTransformer +(90) ProjectExecTransformer Output [5]: [hash(nation#X, o_year#X, 42) AS hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(81) WholeStageCodegenTransformer (X) +(91) WholeStageCodegenTransformer (X) Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: false -(82) ColumnarExchange +(92) ColumnarExchange Input [5]: [hash_partition_key#X, nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [nation#X, o_year#X, sum#X, isEmpty#X], [plan_id=X], [id=#X] -(83) ShuffleQueryStage +(93) ShuffleQueryStage Output [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: X -(84) InputAdapter +(94) InputAdapter Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(85) InputIteratorTransformer +(95) InputIteratorTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(86) RegularHashAggregateExecTransformer +(96) RegularHashAggregateExecTransformer Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(87) WholeStageCodegenTransformer (X) +(97) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(88) ColumnarExchange +(98) VeloxAppendBatches +Input [3]: [nation#X, o_year#X, sum_profit#X] +Arguments: X + +(99) ColumnarExchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X], [id=#X] -(89) ShuffleQueryStage +(100) ShuffleQueryStage Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: X -(90) InputAdapter +(101) InputAdapter Input [3]: [nation#X, o_year#X, sum_profit#X] -(91) InputIteratorTransformer +(102) InputIteratorTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] -(92) SortExecTransformer +(103) SortExecTransformer Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(93) WholeStageCodegenTransformer (X) +(104) WholeStageCodegenTransformer (X) Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: false -(94) VeloxColumnarToRowExec +(105) VeloxColumnarToRowExec Input [3]: [nation#X, o_year#X, sum_profit#X] -(95) Scan parquet +(106) Scan parquet Output [2]: [p_partkey#X, p_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(p_name), StringContains(p_name,green), IsNotNull(p_partkey)] ReadSchema: struct -(96) Filter +(107) Filter Input [2]: [p_partkey#X, p_name#X] Condition : ((isnotnull(p_name#X) AND Contains(p_name#X, green)) AND isnotnull(p_partkey#X)) -(97) Project +(108) Project Output [1]: [p_partkey#X] Input [2]: [p_partkey#X, p_name#X] -(98) Exchange +(109) Exchange Input [1]: [p_partkey#X] Arguments: hashpartitioning(p_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(99) Scan parquet +(110) Scan parquet Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(l_partkey), IsNotNull(l_suppkey), IsNotNull(l_orderkey)] ReadSchema: struct -(100) Filter +(111) Filter Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Condition : ((isnotnull(l_partkey#X) AND isnotnull(l_suppkey#X)) AND isnotnull(l_orderkey#X)) -(101) Exchange +(112) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(102) ShuffledHashJoin +(113) ShuffledHashJoin Left keys [1]: [p_partkey#X] Right keys [1]: [l_partkey#X] Join type: Inner Join condition: None -(103) Project +(114) Project Output [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Input [7]: [p_partkey#X, l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] -(104) Exchange +(115) Exchange Input [6]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X] Arguments: hashpartitioning(l_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(105) Scan parquet +(116) Scan parquet Output [2]: [s_suppkey#X, s_nationkey#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(s_suppkey), IsNotNull(s_nationkey)] ReadSchema: struct -(106) Filter +(117) Filter Input [2]: [s_suppkey#X, s_nationkey#X] Condition : (isnotnull(s_suppkey#X) AND isnotnull(s_nationkey#X)) -(107) Exchange +(118) Exchange Input [2]: [s_suppkey#X, s_nationkey#X] Arguments: hashpartitioning(s_suppkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(108) ShuffledHashJoin +(119) ShuffledHashJoin Left keys [1]: [l_suppkey#X] Right keys [1]: [s_suppkey#X] Join type: Inner Join condition: None -(109) Project +(120) Project Output [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Input [8]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_suppkey#X, s_nationkey#X] -(110) Exchange +(121) Exchange Input [7]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X] Arguments: hashpartitioning(l_suppkey#X, l_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(111) Scan parquet +(122) Scan parquet Output [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(ps_suppkey), IsNotNull(ps_partkey)] ReadSchema: struct -(112) Filter +(123) Filter Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Condition : (isnotnull(ps_suppkey#X) AND isnotnull(ps_partkey#X)) -(113) Exchange +(124) Exchange Input [3]: [ps_partkey#X, ps_suppkey#X, ps_supplycost#X] Arguments: hashpartitioning(ps_suppkey#X, ps_partkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(114) ShuffledHashJoin +(125) ShuffledHashJoin Left keys [2]: [l_suppkey#X, l_partkey#X] Right keys [2]: [ps_suppkey#X, ps_partkey#X] Join type: Inner Join condition: None -(115) Project +(126) Project Output [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Input [10]: [l_orderkey#X, l_partkey#X, l_suppkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_partkey#X, ps_suppkey#X, ps_supplycost#X] -(116) Exchange +(127) Exchange Input [6]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X] Arguments: hashpartitioning(l_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(117) Scan parquet +(128) Scan parquet Output [2]: [o_orderkey#X, o_orderdate#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(o_orderkey)] ReadSchema: struct -(118) Filter +(129) Filter Input [2]: [o_orderkey#X, o_orderdate#X] Condition : isnotnull(o_orderkey#X) -(119) Exchange +(130) Exchange Input [2]: [o_orderkey#X, o_orderdate#X] Arguments: hashpartitioning(o_orderkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(120) ShuffledHashJoin +(131) ShuffledHashJoin Left keys [1]: [l_orderkey#X] Right keys [1]: [o_orderkey#X] Join type: Inner Join condition: None -(121) Project +(132) Project Output [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Input [8]: [l_orderkey#X, l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderkey#X, o_orderdate#X] -(122) Exchange +(133) Exchange Input [6]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X] Arguments: hashpartitioning(s_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(123) Scan parquet +(134) Scan parquet Output [2]: [n_nationkey#X, n_name#X] Batched: true Location: InMemoryFileIndex [*] PushedFilters: [IsNotNull(n_nationkey)] ReadSchema: struct -(124) Filter +(135) Filter Input [2]: [n_nationkey#X, n_name#X] Condition : isnotnull(n_nationkey#X) -(125) Exchange +(136) Exchange Input [2]: [n_nationkey#X, n_name#X] Arguments: hashpartitioning(n_nationkey#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(126) ShuffledHashJoin +(137) ShuffledHashJoin Left keys [1]: [s_nationkey#X] Right keys [1]: [n_nationkey#X] Join type: Inner Join condition: None -(127) Project +(138) Project Output [3]: [n_name#X AS nation#X, year(o_orderdate#X) AS o_year#X, ((l_extendedprice#X * (1 - l_discount#X)) - (ps_supplycost#X * l_quantity#X)) AS amount#X] Input [8]: [l_quantity#X, l_extendedprice#X, l_discount#X, s_nationkey#X, ps_supplycost#X, o_orderdate#X, n_nationkey#X, n_name#X] -(128) HashAggregate +(139) HashAggregate Input [3]: [nation#X, o_year#X, amount#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [partial_sum(amount#X)] Aggregate Attributes [2]: [sum#X, isEmpty#X] Results [4]: [nation#X, o_year#X, sum#X, isEmpty#X] -(129) Exchange +(140) Exchange Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Arguments: hashpartitioning(nation#X, o_year#X, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(130) HashAggregate +(141) HashAggregate Input [4]: [nation#X, o_year#X, sum#X, isEmpty#X] Keys [2]: [nation#X, o_year#X] Functions [1]: [sum(amount#X)] Aggregate Attributes [1]: [sum(amount#X)#X] Results [3]: [nation#X, o_year#X, sum(amount#X)#X AS sum_profit#X] -(131) Exchange +(142) Exchange Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: rangepartitioning(nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST, 1), ENSURE_REQUIREMENTS, [plan_id=X] -(132) Sort +(143) Sort Input [3]: [nation#X, o_year#X, sum_profit#X] Arguments: [nation#X ASC NULLS FIRST, o_year#X DESC NULLS LAST], true, 0 -(133) AdaptiveSparkPlan +(144) AdaptiveSparkPlan Output [3]: [nation#X, o_year#X, sum_profit#X] Arguments: isFinalPlan=true \ No newline at end of file